goPaxan/webActions.go

package paxan

import (
	"encoding/csv"
	"io"
	"log"
	"net/http"
	"net/http/cookiejar"
	"net/url"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/PuerkitoBio/goquery"
	"golang.org/x/text/encoding/charmap"
)

type paxanWebOrder struct {
	InvoiceNumber int
	Type          string
	Date          string
}

type paxanWebOrderEntry struct {
	Position      string
	ArticleNumber int
	Description   string
	ProductLink   string
	Amount        string
	Price         string
	Total         string
	Amount2       string
	Price2        string
	Total2        string
}

// SpricelistEntry - One entry in the spricelist
type SpricelistEntry struct {
	Sortiment    string
	Warengruppe  string
	Untergruppe  string
	Artnr        int
	Itemname1    string
	Itemname2    string
	Hersteller   string
	Qualitaet    string
	Herkunft     string
	Einheit      string
	Aktion       bool
	Menge1       float64
	Einzelpreis1 float64
	Vpepreis1    float64
	Menge2       string
	Einzelpreis2 string
	Vpepreis2    string
	Menge3       string
	Einzelpreis3 string
	Vpepreis3    string
	Menge4       string
	Einzelpreis4 string
	Vpepreis4    string
	Empfvk       float64
	Mwst         float64
	Ean          string
	Glutenfrei   bool
	Laktosefrei  bool
	Vegan        bool
	Regional     bool
	Fairtrade    bool
}

func convertStringToBool(value string) bool {
	if value == "X" {
		return true
	}
	if value == "A" {
		return true
	}
	return false
}

func convertStringToFloat(value string) float64 {
	value = strings.Replace(value, " €", "", -1)
	value = strings.Replace(value, ",", ".", -1)
	valueFloat, _ := strconv.ParseFloat(value, 46)
	return valueFloat
}

func convertStringToInt(value string) int {
	valueInt, _ := strconv.Atoi(value)
	return valueInt
}

func convertStringToTime(value string) time.Time {
	time, err := time.Parse("02.01.06", value)
	if err != nil {
		log.Fatalln(err)
	}
	return time
}

func downloadFile(filepath string, resp *http.Response) error {
	// Create the file
	file, err := os.Create(filepath)
	if err != nil {
		return err
	}
	defer file.Close()

	// Write the body to file
	_, err = io.Copy(file, resp.Body)
	if err != nil {
		return err
	}

	return nil
}

// NewClient - Login
func NewClient(user string, password string) (*http.Client, error) {
	// New http client with cookie
	cookieJar, _ := cookiejar.New(nil)
	client := &http.Client{
		Jar: cookieJar,
	}

	loginvalues := make(url.Values)
	loginvalues.Set("performAction", "processLogin")
	loginvalues.Set("personlogin", user)
	loginvalues.Set("personpwd", password)
	loginvalues.Set("firma", "paxan")
	_, err := client.PostForm("https://www.hakopaxan-shop.de/html/login.html", loginvalues)
	if err != nil {
		return nil, err
	}

	return client, nil
}

// CloseClient - Logout
func CloseClient(client *http.Client) error {
	// Logout
	_, err := client.Get("https://www.hakopaxan-shop.de/html/logout-performAction-processLogout.html")
	if err != nil {
		return err
	}

	return nil
}

// DownloadPricelists - Download both price lists
func DownloadPricelists(client *http.Client) error {
	// Download preisliste
	r, err := client.Get("https://www.hakopaxan-shop.de/csv/preisliste.csv")
	if err != nil {
		return err
	}
	downloadFile(("preisliste-" + time.Now().Format("20060102150405") + ".csv"), r)

	// Download spreisliste
	r, err = client.Get("https://www.hakopaxan-shop.de/csv/spreisliste.csv")
	if err != nil {
		return err
	}
	downloadFile(("spreisliste-" + time.Now().Format("20060102150405") + ".csv"), r)

	return nil
}

// RetrievePricelist - Downloads and converts pricelist into an array
func RetrievePricelist(client *http.Client) ([]SpricelistEntry, error) {
	// Download spreisliste
	resp, err := client.Get("https://www.hakopaxan-shop.de/csv/spreisliste.csv")
	if err != nil {
		return nil, err
	}

	dr := charmap.ISO8859_1.NewDecoder().Reader(resp.Body)

	r := csv.NewReader(dr)
	r.Comma = ';'
	records, err := r.ReadAll()
	if err != nil {
		log.Fatal(err)
	}

	var priceList []SpricelistEntry

	for i, line := range records {
		if i == 0 {
			continue
		}
		var priceListEntry SpricelistEntry

		priceListEntry.Sortiment = line[0]
		priceListEntry.Warengruppe = line[1]
		priceListEntry.Untergruppe = line[2]
		priceListEntry.Artnr = convertStringToInt(line[3])
		priceListEntry.Itemname1 = line[4]
		priceListEntry.Itemname2 = line[5]
		priceListEntry.Hersteller = line[6]
		priceListEntry.Qualitaet = line[7]
		priceListEntry.Herkunft = line[8]
		priceListEntry.Einheit = line[9]
		priceListEntry.Aktion = convertStringToBool(line[10])
		priceListEntry.Menge1 = convertStringToFloat(line[11])
		priceListEntry.Einzelpreis1 = convertStringToFloat(line[12])
		priceListEntry.Vpepreis1 = convertStringToFloat(line[13])
		priceListEntry.Menge2 = line[14]
		priceListEntry.Einzelpreis2 = line[15]
		priceListEntry.Vpepreis2 = line[16]
		priceListEntry.Menge3 = line[17]
		priceListEntry.Einzelpreis3 = line[18]
		priceListEntry.Vpepreis3 = line[19]
		priceListEntry.Menge4 = line[20]
		priceListEntry.Einzelpreis4 = line[21]
		priceListEntry.Vpepreis4 = line[22]
		priceListEntry.Empfvk = convertStringToFloat(line[23])
		priceListEntry.Mwst = convertStringToFloat(line[24])
		priceListEntry.Ean = line[25]
		priceListEntry.Glutenfrei = convertStringToBool(line[26])
		priceListEntry.Laktosefrei = convertStringToBool(line[27])
		priceListEntry.Vegan = convertStringToBool(line[28])
		priceListEntry.Regional = convertStringToBool(line[29])
		priceListEntry.Fairtrade = convertStringToBool(line[30])

		priceList = append(priceList, priceListEntry)
	}

	return priceList, nil

}

func paxanGetOrders(client *http.Client) ([]paxanWebOrder, error) {
	var list []paxanWebOrder
	kvs := make(map[string]string)

	r, err := client.Get("https://www.hakopaxan-shop.de/html/customerDocumentList.html?customerDocumentListId=invoice&requestedPeriod=365")
	if err != nil {
		return nil, err
	}

	doc, err := goquery.NewDocumentFromReader(io.Reader(r.Body))
	if err != nil {
		return nil, err
	}

	var headline []string
	// Find the table header
	doc.Find(".tableHeader").Each(func(i int, s *goquery.Selection) {
		s.Find(".tableCell").Each(func(i int, s *goquery.Selection) {
			title := s.Contents().Text()
			title = strings.Replace(title, "\n", "", -1)
			title = strings.TrimSpace(title)

			headline = append(headline, title)
		})
	})

	// Find the table items
	doc.Find(".tableRowGroup").Each(func(i int, s *goquery.Selection) {
		s.Find(".tableCell").Each(func(j int, s *goquery.Selection) {
			title := "undefined"
			if j < len(headline) {
				title = headline[j]
			}

			text := s.Contents().Text()
			text = strings.Replace(text, "\n", "", -1)
			text = strings.Split(text, ":")[1]
			text = strings.TrimSpace(text)
			kvs[title] = text
			//fmt.Printf("%s - %s\n", title, text)
		})
		//fmt.Println(kvs)

		num, _ := strconv.Atoi(kvs["Beleg-Nr."])
		n := paxanWebOrder{InvoiceNumber: num, Type: kvs["Belegart"], Date: kvs["Datum"]}
		list = append(list, n)
	})

	return list, nil
}

func paxanGetOrderDetails(client *http.Client, orderNumber int) ([]paxanWebOrderEntry, error) {
	var list []paxanWebOrderEntry

	url := "https://www.hakopaxan-shop.de/html/customerDocument-customerDocumentId-" + strconv.Itoa(orderNumber) + "-customerDocumentListId-invoice-requestedPeriod-365.html"
	r, err := client.Get(url)
	if err != nil {
		return nil, err
	}

	doc, err := goquery.NewDocumentFromReader(io.Reader(r.Body))
	if err != nil {
		return nil, err
	}

	// Go through table row by row
	doc.Find(".tableRowGroup").Each(func(i int, s *goquery.Selection) {
		var row []string
		s.Find(".tableCell").Each(func(j int, s *goquery.Selection) {
			text := s.Contents().Text()
			// Column 3 contains some more info
			if j == 2 {
				text = s.Find(".lineItemName").Text()
				// Search for the link to the product page
				linkTag := s.Find("a")
				link, _ := linkTag.Attr("href")
				row = append(row, link)
			}

			text = strings.Replace(text, "\n", "", -1)
			text = strings.TrimSpace(text)
			row = append(row, text)

		})

		num, _ := strconv.Atoi(row[1])
		n := paxanWebOrderEntry{Position: row[0],
			ArticleNumber: num,
			Description:   row[3],
			ProductLink:   row[2],
			Amount:        row[4],
			Price:         row[5],
			Total:         row[6],
			Amount2:       row[7],
			Price2:        row[8],
			Total2:        row[9]}
		list = append(list, n)
	})

	return list, nil
}

// ExportNewOrders - Export new orders into orders.csv
func ExportNewOrders(client *http.Client, lastOrder int) error {
	f, err := os.OpenFile("orders.csv", os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
	if err != nil {
		return err
	}
	defer f.Close()

	orders, err := paxanGetOrders(client)
	if err != nil {
		return err
	}

	i, j := 0, 0
	for _, order := range orders {
		if order.InvoiceNumber <= lastOrder {
			continue
		}

		orderDetails, err := paxanGetOrderDetails(client, order.InvoiceNumber)
		if err != nil {
			return err
		}

		for _, orderDetail := range orderDetails {
			text := strconv.Itoa(order.InvoiceNumber) + ";" +
				order.Type + ";" +
				order.Date + ";" +

				orderDetail.Position + ";" +
				strconv.Itoa(orderDetail.ArticleNumber) + ";" +
				orderDetail.Description + ";" +
				orderDetail.ProductLink + ";" +
				orderDetail.Amount + ";" +
				orderDetail.Price + ";" +
				orderDetail.Total + ";" +
				orderDetail.Amount2 + ";" +
				orderDetail.Price2 + ";" +
				orderDetail.Total2 + "\n"

			_, err = f.WriteString(text)
			if err != nil {
				return err
			}

			j++
		}

		i++
	}

	log.Printf("Added %d order(s) with %d article(s)", i, j)

	return nil
}