talksik
190 lines
4.3 KiB
Go
190 lines
4.3 KiB
Go
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"fmt"
|
|
"errors"
|
|
"golang.org/x/net/html"
|
|
"golang.org/x/net/html/atom"
|
|
"io"
|
|
"io/fs"
|
|
"net/http"
|
|
"os"
|
|
"strings"
|
|
"encoding/json"
|
|
)
|
|
|
|
type AiMessage struct {
|
|
Role string `json:"role"`
|
|
Content string `json:"content"`
|
|
}
|
|
type AiRequest struct {
|
|
Messages []AiMessage `json:"messages"`
|
|
}
|
|
type AiChoice struct {
|
|
Message AiMessage `json:"message"`
|
|
}
|
|
type AiResponse struct {
|
|
Choices []AiChoice `json:"choices"`
|
|
}
|
|
|
|
const llamaServerUrl = "http://asus:5050/v1/chat/completions"
|
|
|
|
type Action struct {
|
|
Node *html.Node
|
|
Href string
|
|
}
|
|
|
|
func main() {
|
|
if len(os.Args) < 2 {
|
|
panic("cmon man")
|
|
}
|
|
website := os.Args[1]
|
|
fmt.Printf("parsing website: %s\n", website)
|
|
|
|
resp, err := http.Get(website)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
|
|
defer resp.Body.Close()
|
|
body, err := io.ReadAll(resp.Body)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
|
|
bodyContents := string(body)
|
|
actionsOutput, err := generateActionsOutput(bodyContents, website)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
|
|
err = os.WriteFile("result.html", []byte(actionsOutput), fs.ModeAppend)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
|
|
dumbWebOutput, err := generateDumbOutput(bodyContents)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
|
|
err = os.WriteFile("result2.html", []byte(dumbWebOutput), fs.ModeAppend)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
}
|
|
|
|
func generateDumbOutput(bodyContents string) (string, error) {
|
|
request := AiRequest{
|
|
Messages: []AiMessage{
|
|
{
|
|
Role: "developer",
|
|
Content: "Your goal is to convert given html content into a simplistic version of the web page provided. This means it should look so essentialist without any bloat of the original provided content. it should retain core interactive elements for the web page to remain usable. Only return ready-to-use HTML, no primer message before the outputted html. Discard any non-essential functionality within the web page and just convert what is essential.",
|
|
},
|
|
{
|
|
Role: "user",
|
|
Content: bodyContents,
|
|
},
|
|
},
|
|
}
|
|
data, err := json.Marshal(request)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
responseData, err := http.Post(llamaServerUrl, "application/json", bytes.NewBuffer(data))
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
defer responseData.Body.Close()
|
|
responseBody, err := io.ReadAll(responseData.Body)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
var response AiResponse
|
|
err = json.Unmarshal(responseBody, &response)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
if len(response.Choices) == 0 {
|
|
return "", errors.New("no choices in response from ai server")
|
|
}
|
|
|
|
outputHtml := response.Choices[0].Message.Content
|
|
fmt.Printf("Here is the result html: %s\n", outputHtml)
|
|
return outputHtml, nil
|
|
}
|
|
|
|
func generateActionsOutput(bodyContents string, website string) (string, error) {
|
|
// parse the specific <a> tags with html stdlib and button
|
|
doc, err := html.Parse(strings.NewReader(bodyContents))
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
// find the descendants that are <a>, find hrefs
|
|
actions := []Action{}
|
|
for node := range doc.Descendants() {
|
|
if node.Type == html.ElementNode {
|
|
if node.DataAtom == atom.A {
|
|
for _, attr := range node.Attr {
|
|
if attr.Key == "href" {
|
|
href := fmt.Sprintf("%s%s", website, attr.Val)
|
|
action := Action{
|
|
Href: href,
|
|
Node: node,
|
|
}
|
|
actions = append(actions, action)
|
|
break
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
fmt.Printf("found %d actions\n", len(actions))
|
|
|
|
output := `<!DOCTYPE html>
|
|
<div style="display:flex; flex-direction: column;">`
|
|
for i, action := range actions {
|
|
fmt.Printf("action %d: %s\n", i, action.Href)
|
|
|
|
output += fmt.Sprintf(`<a href="%s" style="padding: 10px;">`, action.Href)
|
|
|
|
// get the childnodes, and use
|
|
children := action.Node.ChildNodes()
|
|
for child := range children {
|
|
var buf bytes.Buffer
|
|
err := html.Render(&buf, child)
|
|
if err != nil {
|
|
fmt.Printf("err in rendering first child: %s\n", err)
|
|
continue
|
|
}
|
|
output += fmt.Sprintf(`%s`, buf.String())
|
|
}
|
|
output += `</a>`
|
|
}
|
|
output += `</div></html>`
|
|
return output, nil
|
|
}
|
|
|
|
// Traverse all children of node until Data is of type text
|
|
func extractNodeText(node *html.Node) string {
|
|
if node.Type == html.TextNode {
|
|
return node.Data
|
|
}
|
|
|
|
if node.FirstChild == nil {
|
|
return ""
|
|
}
|
|
|
|
return extractNodeText(node.FirstChild)
|
|
}
|
|
|
|
func isValidTextNode(data string) bool {
|
|
exNewLines := strings.Trim(data, "\n")
|
|
return strings.Trim(exNewLines, " ") != ""
|
|
}
|