package paxan import ( "encoding/csv" "io" "log" "net/http" "net/http/cookiejar" "net/url" "os" "strconv" "strings" "time" "github.com/PuerkitoBio/goquery" "golang.org/x/text/encoding/charmap" ) type paxanWebOrder struct { InvoiceNumber int Type string Date string } type paxanWebOrderEntry struct { Position string ArticleNumber int Description string ProductLink string Amount string Price string Total string Amount2 string Price2 string Total2 string } // SpricelistEntry - One entry in the spricelist type SpricelistEntry struct { Sortiment string Warengruppe string Untergruppe string Artnr int Itemname1 string Itemname2 string Hersteller string Qualitaet string Herkunft string Einheit string Aktion bool Menge1 float64 Einzelpreis1 float64 Vpepreis1 float64 Menge2 string Einzelpreis2 string Vpepreis2 string Menge3 string Einzelpreis3 string Vpepreis3 string Menge4 string Einzelpreis4 string Vpepreis4 string Empfvk float64 Mwst float64 Ean string Glutenfrei bool Laktosefrei bool Vegan bool Regional bool Fairtrade bool } func convertStringToBool(value string) bool { if value == "X" { return true } if value == "A" { return true } return false } func convertStringToFloat(value string) float64 { value = strings.Replace(value, " €", "", -1) value = strings.Replace(value, ",", ".", -1) valueFloat, _ := strconv.ParseFloat(value, 46) return valueFloat } func convertStringToInt(value string) int { valueInt, _ := strconv.Atoi(value) return valueInt } func convertStringToTime(value string) time.Time { time, err := time.Parse("02.01.06", value) if err != nil { log.Fatalln(err) } return time } func downloadFile(filepath string, resp *http.Response) error { // Create the file file, err := os.Create(filepath) if err != nil { return err } defer file.Close() // Write the body to file _, err = io.Copy(file, resp.Body) if err != nil { return err } return nil } // NewClient - Login func NewClient(user string, password string) (*http.Client, error) { // New http client with cookie cookieJar, _ := cookiejar.New(nil) client := &http.Client{ Jar: cookieJar, } loginvalues := make(url.Values) loginvalues.Set("performAction", "processLogin") loginvalues.Set("personlogin", user) loginvalues.Set("personpwd", password) loginvalues.Set("firma", "paxan") _, err := client.PostForm("https://www.hakopaxan-shop.de/html/login.html", loginvalues) if err != nil { return nil, err } return client, nil } // CloseClient - Logout func CloseClient(client *http.Client) error { // Logout _, err := client.Get("https://www.hakopaxan-shop.de/html/logout-performAction-processLogout.html") if err != nil { return err } return nil } // DownloadPricelists - Download both price lists func DownloadPricelists(client *http.Client) error { // Download preisliste r, err := client.Get("https://www.hakopaxan-shop.de/csv/preisliste.csv") if err != nil { return err } downloadFile(("preisliste-" + time.Now().Format("20060102150405") + ".csv"), r) // Download spreisliste r, err = client.Get("https://www.hakopaxan-shop.de/csv/spreisliste.csv") if err != nil { return err } downloadFile(("spreisliste-" + time.Now().Format("20060102150405") + ".csv"), r) return nil } // RetrievePricelist - Downloads and converts pricelist into an array func RetrievePricelist(client *http.Client) ([]SpricelistEntry, error) { // Download spreisliste resp, err := client.Get("https://www.hakopaxan-shop.de/csv/spreisliste.csv") if err != nil { return nil, err } dr := charmap.ISO8859_1.NewDecoder().Reader(resp.Body) r := csv.NewReader(dr) r.Comma = ';' records, err := r.ReadAll() if err != nil { log.Fatal(err) } var priceList []SpricelistEntry for i, line := range records { if i == 0 { continue } var priceListEntry SpricelistEntry priceListEntry.Sortiment = line[0] priceListEntry.Warengruppe = line[1] priceListEntry.Untergruppe = line[2] priceListEntry.Artnr = convertStringToInt(line[3]) priceListEntry.Itemname1 = line[4] priceListEntry.Itemname2 = line[5] priceListEntry.Hersteller = line[6] priceListEntry.Qualitaet = line[7] priceListEntry.Herkunft = line[8] priceListEntry.Einheit = line[9] priceListEntry.Aktion = convertStringToBool(line[10]) priceListEntry.Menge1 = convertStringToFloat(line[11]) priceListEntry.Einzelpreis1 = convertStringToFloat(line[12]) priceListEntry.Vpepreis1 = convertStringToFloat(line[13]) priceListEntry.Menge2 = line[14] priceListEntry.Einzelpreis2 = line[15] priceListEntry.Vpepreis2 = line[16] priceListEntry.Menge3 = line[17] priceListEntry.Einzelpreis3 = line[18] priceListEntry.Vpepreis3 = line[19] priceListEntry.Menge4 = line[20] priceListEntry.Einzelpreis4 = line[21] priceListEntry.Vpepreis4 = line[22] priceListEntry.Empfvk = convertStringToFloat(line[23]) priceListEntry.Mwst = convertStringToFloat(line[24]) priceListEntry.Ean = line[25] priceListEntry.Glutenfrei = convertStringToBool(line[26]) priceListEntry.Laktosefrei = convertStringToBool(line[27]) priceListEntry.Vegan = convertStringToBool(line[28]) priceListEntry.Regional = convertStringToBool(line[29]) priceListEntry.Fairtrade = convertStringToBool(line[30]) priceList = append(priceList, priceListEntry) } return priceList, nil } func paxanGetOrders(client *http.Client) ([]paxanWebOrder, error) { var list []paxanWebOrder kvs := make(map[string]string) r, err := client.Get("https://www.hakopaxan-shop.de/html/customerDocumentList.html?customerDocumentListId=invoice&requestedPeriod=365") if err != nil { return nil, err } doc, err := goquery.NewDocumentFromReader(io.Reader(r.Body)) if err != nil { return nil, err } var headline []string // Find the table header doc.Find(".tableHeader").Each(func(i int, s *goquery.Selection) { s.Find(".tableCell").Each(func(i int, s *goquery.Selection) { title := s.Contents().Text() title = strings.Replace(title, "\n", "", -1) title = strings.TrimSpace(title) headline = append(headline, title) }) }) // Find the table items doc.Find(".tableRowGroup").Each(func(i int, s *goquery.Selection) { s.Find(".tableCell").Each(func(j int, s *goquery.Selection) { title := "undefined" if j < len(headline) { title = headline[j] } text := s.Contents().Text() text = strings.Replace(text, "\n", "", -1) text = strings.Split(text, ":")[1] text = strings.TrimSpace(text) kvs[title] = text //fmt.Printf("%s - %s\n", title, text) }) //fmt.Println(kvs) num, _ := strconv.Atoi(kvs["Beleg-Nr."]) n := paxanWebOrder{InvoiceNumber: num, Type: kvs["Belegart"], Date: kvs["Datum"]} list = append(list, n) }) return list, nil } func paxanGetOrderDetails(client *http.Client, orderNumber int) ([]paxanWebOrderEntry, error) { var list []paxanWebOrderEntry url := "https://www.hakopaxan-shop.de/html/customerDocument-customerDocumentId-" + strconv.Itoa(orderNumber) + "-customerDocumentListId-invoice-requestedPeriod-365.html" r, err := client.Get(url) if err != nil { return nil, err } doc, err := goquery.NewDocumentFromReader(io.Reader(r.Body)) if err != nil { return nil, err } // Go through table row by row doc.Find(".tableRowGroup").Each(func(i int, s *goquery.Selection) { var row []string s.Find(".tableCell").Each(func(j int, s *goquery.Selection) { text := s.Contents().Text() // Column 3 contains some more info if j == 2 { text = s.Find(".lineItemName").Text() // Search for the link to the product page linkTag := s.Find("a") link, _ := linkTag.Attr("href") row = append(row, link) } text = strings.Replace(text, "\n", "", -1) text = strings.TrimSpace(text) row = append(row, text) }) num, _ := strconv.Atoi(row[1]) n := paxanWebOrderEntry{Position: row[0], ArticleNumber: num, Description: row[3], ProductLink: row[2], Amount: row[4], Price: row[5], Total: row[6], Amount2: row[7], Price2: row[8], Total2: row[9]} list = append(list, n) }) return list, nil } // ExportNewOrders - Export new orders into orders.csv func ExportNewOrders(client *http.Client, lastOrder int) error { f, err := os.OpenFile("orders.csv", os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600) if err != nil { return err } defer f.Close() orders, err := paxanGetOrders(client) if err != nil { return err } i, j := 0, 0 for _, order := range orders { if order.InvoiceNumber <= lastOrder { continue } orderDetails, err := paxanGetOrderDetails(client, order.InvoiceNumber) if err != nil { return err } for _, orderDetail := range orderDetails { text := strconv.Itoa(order.InvoiceNumber) + ";" + order.Type + ";" + order.Date + ";" + orderDetail.Position + ";" + strconv.Itoa(orderDetail.ArticleNumber) + ";" + orderDetail.Description + ";" + orderDetail.ProductLink + ";" + orderDetail.Amount + ";" + orderDetail.Price + ";" + orderDetail.Total + ";" + orderDetail.Amount2 + ";" + orderDetail.Price2 + ";" + orderDetail.Total2 + "\n" _, err = f.WriteString(text) if err != nil { return err } j++ } i++ } log.Printf("Added %d order(s) with %d article(s)", i, j) return nil }