package main import ( "bytes" "fmt" "errors" "golang.org/x/net/html" "golang.org/x/net/html/atom" "io" "io/fs" "net/http" "os" "strings" "encoding/json" ) type AiMessage struct { Role string `json:"role"` Content string `json:"content"` } type AiRequest struct { Messages []AiMessage `json:"messages"` } type AiChoice struct { Message AiMessage `json:"message"` } type AiResponse struct { Choices []AiChoice `json:"choices"` } const llamaServerUrl = "http://asus:5050/v1/chat/completions" type Action struct { Node *html.Node Href string } func main() { if len(os.Args) < 2 { panic("cmon man") } website := os.Args[1] fmt.Printf("parsing website: %s\n", website) resp, err := http.Get(website) if err != nil { panic(err) } defer resp.Body.Close() body, err := io.ReadAll(resp.Body) if err != nil { panic(err) } bodyContents := string(body) actionsOutput, err := generateActionsOutput(bodyContents, website) if err != nil { panic(err) } err = os.WriteFile("result.html", []byte(actionsOutput), fs.ModeAppend) if err != nil { panic(err) } dumbWebOutput, err := generateDumbOutput(bodyContents) if err != nil { panic(err) } err = os.WriteFile("result2.html", []byte(dumbWebOutput), fs.ModeAppend) if err != nil { panic(err) } } func generateDumbOutput(bodyContents string) (string, error) { request := AiRequest{ Messages: []AiMessage{ { Role: "developer", Content: "Your goal is to convert given html content into a simplistic version of the web page provided. This means it should look so essentialist and use tailwind and simple components without any bloat of the original provided content. Only return ready to use HTML.", }, { Role: "user", Content: bodyContents, }, }, } data, err := json.Marshal(request) if err != nil { return "", err } responseData, err := http.Post(llamaServerUrl, "application/json", bytes.NewBuffer(data)) if err != nil { return "", err } defer responseData.Body.Close() responseBody, err := io.ReadAll(responseData.Body) if err != nil { return "", err } var response AiResponse err = json.Unmarshal(responseBody, &response) if err != nil { return "", err } if len(response.Choices) == 0 { return "", errors.New("no choices in response from ai server") } outputHtml := response.Choices[0].Message.Content fmt.Printf("Here is the result html: %s\n", outputHtml) return outputHtml, nil } func generateActionsOutput(bodyContents string, website string) (string, error) { // parse the specific tags with html stdlib and button doc, err := html.Parse(strings.NewReader(bodyContents)) if err != nil { return "", err } // find the descendants that are , find hrefs actions := []Action{} for node := range doc.Descendants() { if node.Type == html.ElementNode { if node.DataAtom == atom.A { for _, attr := range node.Attr { if attr.Key == "href" { href := fmt.Sprintf("%s%s", website, attr.Val) action := Action{ Href: href, Node: node, } actions = append(actions, action) break } } } } } fmt.Printf("found %d actions\n", len(actions)) output := `

` for i, action := range actions { fmt.Printf("action %d: %s\n", i, action.Href) output += fmt.Sprintf(``, action.Href) // get the childnodes, and use children := action.Node.ChildNodes() for child := range children { var buf bytes.Buffer err := html.Render(&buf, child) if err != nil { fmt.Printf("err in rendering first child: %s\n", err) continue } output += fmt.Sprintf(`%s`, buf.String()) } output += `` } output += `

` return output, nil } // Traverse all children of node until Data is of type text func extractNodeText(node *html.Node) string { if node.Type == html.TextNode { return node.Data } if node.FirstChild == nil { return "" } return extractNodeText(node.FirstChild) } func isValidTextNode(data string) bool { exNewLines := strings.Trim(data, "\n") return strings.Trim(exNewLines, " ") != "" }