Nim и Go против Wikipedia22.06.2015 23:18

package main

import (
        "flag"
        "fmt"
        "github.com/PuerkitoBio/goquery"
        "log"
        "net/url"
        "strings"
)

const stopTerm string = "#stop#"

var (
        lang      string
        printMode int
)

func findLinks(url string) []string {
        result := make([]string, 0)

        doc, err := goquery.NewDocument(fmt.Sprintf("https://%s.wikipedia.org%s", lang, url))
        if err != nil {
                log.Fatal(err)
        }

        doc.Find("a").Each(func(i int, s *goquery.Selection) {
                href, _ := s.Attr("href")
                if strings.HasPrefix(href, "/wiki/") && !strings.Contains(href, ":") && !strings.Contains(href, "#") {
                        result = append(result, href)
                }
        })

        return result
}

func printTerm(term string, mode int) {
        if printMode == mode {
                val, err := url.QueryUnescape(term)
                if err != nil {
                        log.Fatal(err)
                }
                fmt.Printf("%s\n", val)
        }
}

func printParent(term string, dict map[string]string) {
        val, _ := url.QueryUnescape(term)
        fmt.Printf("\n\n>> Found: %s\n", val)
        n := dict[term]
        for n != stopTerm {
                val, _ := url.QueryUnescape(n)
                fmt.Printf("parent: %s\n", val)
                n = dict[n]
        }
}

func searchPath(begin string, end string) {
        dict := make(map[string]string)
        dict[begin] = stopTerm

        queue := make(chan string, 10000000)
        queue <- begin

        for {
                currentNode := <-queue
                printTerm(currentNode, 1)
                for _, v := range findLinks(currentNode) {
                        if _, ok := dict[v]; !ok {
                                dict[v] = currentNode
                                if v == end {
                                        printParent(v, dict)
                                        return
                                }

                                queue <- v

                                printTerm(v, 0)
                        }
                }
        }
}

func main() {
        beginTerm := flag.String("b", "Sort", "Begin wiki term")
        endTerm := flag.String("e", "SAP", "Finish wiki term")
        langFlag := flag.String("l", "ru", ".wikipedia.org")
        printModeFlag := flag.Int("p", 0, "Print mode 0 - print all; 1- print current term; 2 - silent")
        flag.Parse()

        lang = *langFlag
        printMode = *printModeFlag

        searchPath("/wiki/"+url.QueryEscape(*beginTerm), "/wiki/"+url.QueryEscape(*endTerm))
}

import httpclient, streams, htmlparser,
    xmltree, strtabs, strutils,
    queues, cgi, os

proc printTerm(term: string) = 
    echo decodeUrl(term)

proc findLinks(url: string) : seq[string] =
    result = @[]
    let html = getContent(r"https://ru.wikipedia.org" & url)
    for a in parseHtml(newStringStream(html)).findAll("a"):
        let href = a.attrs["href"]
        if href.startsWith("/wiki/") and not href.contains({':', '#'}):
            result.add(href)

proc printParent(term: string, dict: StringTableRef) =
    printTerm("\nFound! " & term)
    var n = dict[term]
    while n != "#stop#":
        printTerm("parent: " & n)
        n = dict[n]

proc searchPath(beginTerm: string, endTerm: string) = 
    var dict = newStringTable(modeCaseInsensitive)
    var queue = initQueue[string]()

    dict[beginTerm] = "#stop#"
    queue.add(beginTerm)

    while true:
        var currentNode = queue.dequeue  
        printTerm(currentNode)
        for a in findLinks(currentNode):
            if not dict.hasKey(a):
                dict[a] = currentNode
                if a == endTerm:
                    printParent(a, dict)
                    return
                queue.add(a)

let args = commandLineParams()
case args.len:
    of 2:
        searchPath("/wiki/" & encodeUrl(args[0]), "/wiki/" & encodeUrl(args[1]))
    of 0:
        searchPath("/wiki/Sort", "/wiki/SAP")
    else:   
        echo r"use >>program  "