package main import ( "bytes" "fmt" "errors" "golang.org/x/net/html" "golang.org/x/net/html/atom" "io" "io/fs" "net/http" "os" "strings" "encoding/json" ) type AiMessage struct { Role string `json:"role"` Content string `json:"content"` } type AiRequest struct { Messages []AiMessage `json:"messages"` } type AiChoice struct { Message AiMessage `json:"message"` } type AiResponse struct { Choices []AiChoice `json:"choices"` } const llamaServerUrl = "http://asus:5050/v1/chat/completions" type Action struct { Node *html.Node Href string } func main() { if len(os.Args) < 2 { panic("cmon man") } website := os.Args[1] fmt.Printf("parsing website: %s\n", website) resp, err := http.Get(website) if err != nil { panic(err) } defer resp.Body.Close() body, err := io.ReadAll(resp.Body) if err != nil { panic(err) } bodyContents := string(body) actionsOutput, err := generateActionsOutput(bodyContents, website) if err != nil { panic(err) } err = os.WriteFile("result.html", []byte(actionsOutput), fs.ModeAppend) if err != nil { panic(err) } dumbWebOutput, err := generateDumbOutput(bodyContents) if err != nil { panic(err) } err = os.WriteFile("result2.html", []byte(dumbWebOutput), fs.ModeAppend) if err != nil { panic(err) } } func generateDumbOutput(bodyContents string) (string, error) { request := AiRequest{ Messages: []AiMessage{ { Role: "developer", Content: "Your goal is to convert given html content into a simplistic version of the web page provided. This means it should look so essentialist without any bloat of the original provided content. it should retain core interactive elements for the web page to remain usable. Only return ready-to-use HTML, no primer message before the outputted html. Discard any non-essential functionality within the web page and just convert what is essential.", }, { Role: "user", Content: bodyContents, }, }, } data, err := json.Marshal(request) if err != nil { return "", err } responseData, err := http.Post(llamaServerUrl, "application/json", bytes.NewBuffer(data)) if err != nil { return "", err } defer responseData.Body.Close() responseBody, err := io.ReadAll(responseData.Body) if err != nil { return "", err } var response AiResponse err = json.Unmarshal(responseBody, &response) if err != nil { return "", err } if len(response.Choices) == 0 { return "", errors.New("no choices in response from ai server") } outputHtml := response.Choices[0].Message.Content fmt.Printf("Here is the result html: %s\n", outputHtml) return outputHtml, nil } func generateActionsOutput(bodyContents string, website string) (string, error) { // parse the specific tags with html stdlib and button doc, err := html.Parse(strings.NewReader(bodyContents)) if err != nil { return "", err } // find the descendants that are , find hrefs actions := []Action{} for node := range doc.Descendants() { if node.Type == html.ElementNode { if node.DataAtom == atom.A { for _, attr := range node.Attr { if attr.Key == "href" { href := fmt.Sprintf("%s%s", website, attr.Val) action := Action{ Href: href, Node: node, } actions = append(actions, action) break } } } } } fmt.Printf("found %d actions\n", len(actions)) output := `

` for i, action := range actions { fmt.Printf("action %d: %s\n", i, action.Href) output += fmt.Sprintf(``, action.Href) // get the childnodes, and use children := action.Node.ChildNodes() for child := range children { var buf bytes.Buffer err := html.Render(&buf, child) if err != nil { fmt.Printf("err in rendering first child: %s\n", err) continue } output += fmt.Sprintf(`%s`, buf.String()) } output += `` } output += `

` return output, nil } // Traverse all children of node until Data is of type text func extractNodeText(node *html.Node) string { if node.Type == html.TextNode { return node.Data } if node.FirstChild == nil { return "" } return extractNodeText(node.FirstChild) } func isValidTextNode(data string) bool { exNewLines := strings.Trim(data, "\n") return strings.Trim(exNewLines, " ") != "" }