ai conversation (#49)

* capture audio with alsa

* add brainstorm thoughts

* use basecamp project for managing this project

* add libcurl basic example

* write pcm data to file

* transcribe audio file with deepgram

* transcribe raw audio

* hit anthropic API for ai question

* create full flow with stt, anthropic, tts, and playback

This organizes some components like deepgram into own module, and also
allows gets the full flow to work every time we run the program.

* organize audio and intelligence modules

* organize header files into include directory

* docs: add readme

* docs: explain future work

* docs: add disclaimer about hardcoded audio params

.gitmodules

@@ -1,3 +0,0 @@
-[submodule "stream/thirdparty/llama.cpp"]
-	path = stream/thirdparty/llama.cpp
-	url = https://github.com/ggml-org/llama.cpp

ai-conversation/.gitignore

@@ -0,0 +1,2 @@
+main
+.cache

ai-conversation/Makefile

@@ -0,0 +1,14 @@
+CC = gcc
+CFLAGS = -Wall -g -Iinclude
+TARGET = main
+SRC = main.c deepgram_client.c utils.c intelligence.c audio.c
+LIBS = -lavdevice -lavformat -lavcodec -lavutil -lasound -lcurl
+
+$(TARGET): $(SRC)
+	$(CC) $(CFLAGS) -o $(TARGET) $(SRC) $(LIBS)
+
+clean:
+	rm -f *.o $(TARGET)
+
+install-deps:
+	sudo apt install build-essential libavformat-dev libavdevice-dev libavcodec-dev libavutil-dev libasound2-dev

ai-conversation/README.md

@@ -0,0 +1,34 @@
+# Conversation
+This proof of concept is for implementing conversational loop with AI at a low level with raw C.
+
+## How it works
+This simple program:
+1. Captures audio samples for x seconds into a local file using alsa
+2. Sends the raw pcm data to Deepgram STT API for transcription
+3. Uses the transcript to call Anthropic to get an assistant's response
+4. Converts Anthropic's response to audio using Deepgram's TTS
+5. Plays back the generated audio from Deepgram using alsa
+6. Quits. _There is no conversational loop at the time of this writing._
+
+## Requirements
+Platform: the project is meant for Linux x64. It was successfully run on arch linux, with no guarantees otherwise.
+
+* libcurl
+* build-essential
+* libasound2-dev
+
+_See more deps in Makefile `install-deps` command._
+
+## Getting started
+1. Add `DEEPGRAM_API_KEY` and `ANTHROPIC_API_KEY` to `.bashrc` as environment variables.
+2. Build & run
+```sh
+make
+./main
+```
+
+## Audio configuration
+Please see the `audio.h` and `audio.c` as the params for alsa were hardcoded based on my local arch linux x64 machine. You may have to adjust it based on your setup (e.g. you may have a mic with 2 channels/stereo, which would change the size of each frame).
+
+## Future work
+Add an event loop, maybe add multi-threading, so that we can interrupt the AI, ask follow-up questions, and make it sound like a back and forth conversation.

ai-conversation/audio.c

@@ -0,0 +1,215 @@
+#include "audio.h"
+
+static void print_pcm_state(snd_pcm_t *pcm_handle) {
+  snd_pcm_state_t state = snd_pcm_state(pcm_handle);
+  const char *state_name = snd_pcm_state_name(state);
+  printf("State of pcm handle: %s\n", state_name);
+}
+
+int audio_play(char *buf, size_t byte_count) {
+  printf("going to play audio of %d bytes of data\n", (int)byte_count);
+
+  snd_pcm_t *pcm_handle;
+  // The total frames in the buffer is how many bytes divided by 2 because
+  // of linear16 encoding, one channel
+  size_t total_frames = byte_count / 2;
+
+  int err;
+  err = snd_pcm_open(&pcm_handle, ALSA_SPEAKER_HARDWARE_DEVICE,
+                     SND_PCM_STREAM_PLAYBACK, 0);
+  if (err < 0) {
+    fprintf(stderr, "unable to open playback device");
+    return err;
+  }
+  print_pcm_state(pcm_handle);
+
+  const char *name = snd_pcm_name(pcm_handle);
+  printf("found this device: %s\n", name);
+
+  snd_pcm_info_t *pcm_info;
+  snd_pcm_info_malloc(&pcm_info);
+  err = snd_pcm_info(pcm_handle, pcm_info);
+  if (err < 0) {
+    snd_pcm_close(pcm_handle);
+    printf("unable to get info of pcm device");
+    return -1;
+  }
+  name = snd_pcm_info_get_name(pcm_info);
+  if (err < 0) {
+    printf("unable to get card info");
+    snd_pcm_close(pcm_handle);
+    snd_pcm_info_free(pcm_info);
+    return -1;
+  }
+  printf("got pcm name: %s \n", name);
+
+  print_pcm_state(pcm_handle);
+  snd_pcm_info_free(pcm_info);
+  if ((err = snd_pcm_set_params(pcm_handle, SND_PCM_FORMAT_S16_LE,
+                                SND_PCM_ACCESS_RW_INTERLEAVED, 1, 48000, 1,
+                                500000)) < 0) { /* 0.5sec */
+    printf("Playback open error: %s\n", snd_strerror(err));
+    exit(EXIT_FAILURE);
+  }
+
+  // we want to read in a chunk of data and writei to the pcm
+  size_t frames_written = 0;
+  size_t frames_in_each_chunk = 1024; // how many frames to read per chunk (2048
+                                      // bytes since each frame is 2 bytes)
+  while (frames_written < total_frames) {
+    size_t frames_to_write =
+        frames_in_each_chunk < (total_frames - frames_written)
+            ? frames_in_each_chunk
+            : (total_frames - frames_written);
+    // We move the pointer x2 because each frame is 2 bytes so the next audio
+    // sample is 16 bits forward, not 8
+    char *start_ptr = buf + (frames_written * 2);
+    snd_pcm_sframes_t result =
+        snd_pcm_writei(pcm_handle, start_ptr, frames_to_write);
+    if (result < 0) {
+      result = snd_pcm_recover(pcm_handle, result, 0);
+      if (result < 0) {
+        printf("snd_pcm_writei failed: %s\n", snd_strerror(result));
+        break;
+      }
+    }
+
+    if (result > 0 && result < frames_to_write) {
+      printf("Short write \n");
+    }
+
+    frames_written += result;
+  }
+
+  /* pass the remaining samples, otherwise they're dropped in close */
+  err = snd_pcm_drain(pcm_handle);
+  if (err < 0) {
+    printf("snd_pcm_drain failed: %s\n", snd_strerror(err));
+  }
+  snd_pcm_close(pcm_handle);
+
+  return 0;
+}
+
+char *audio_capture(uint seconds) {
+  char *output_file = "output.pcm";
+
+  snd_pcm_t *pcm_handle = NULL;
+
+  int err;
+  err = snd_pcm_open(&pcm_handle, ALSA_MIC_HARDWARE_DEVICE,
+                     SND_PCM_STREAM_CAPTURE, 0);
+  if (err < 0) {
+    printf("unable to open pcm device");
+    return NULL;
+  }
+  print_pcm_state(pcm_handle);
+
+  const char *name = snd_pcm_name(pcm_handle);
+  printf("found this device: %s\n", name);
+
+  snd_pcm_info_t *pcm_info;
+  snd_pcm_info_malloc(&pcm_info);
+  err = snd_pcm_info(pcm_handle, pcm_info);
+  if (err < 0) {
+    snd_pcm_close(pcm_handle);
+    printf("unable to get info of pcm device");
+    return NULL;
+  }
+  name = snd_pcm_info_get_name(pcm_info);
+  if (err < 0) {
+    printf("unable to get card info");
+    snd_pcm_close(pcm_handle);
+    snd_pcm_info_free(pcm_info);
+    return NULL;
+  }
+  printf("got pcm name: %s \n", name);
+
+  print_pcm_state(pcm_handle);
+  snd_pcm_info_free(pcm_info);
+
+  snd_pcm_hw_params_t *params;
+  err = snd_pcm_hw_params_malloc(&params);
+  if (err < 0) {
+    fprintf(stderr, "unable to mallow hw params");
+    return NULL;
+  }
+  err = snd_pcm_hw_params_any(pcm_handle, params);
+  if (err < 0) {
+    fprintf(stderr, "unable to find ranges for hw device");
+    snd_pcm_close(pcm_handle);
+    snd_pcm_hw_params_free(params);
+    return NULL;
+  }
+
+  uint period_size = 1024;
+  uint buffer_size = 4096;
+  uint channel_count = 1;
+  uint bytes_per_sample = 2; // FORMAT_S16_LE
+  uint samples_per_second = 48000;
+  snd_pcm_hw_params_set_access(pcm_handle, params,
+                               SND_PCM_ACCESS_RW_INTERLEAVED);
+  snd_pcm_hw_params_set_channels(pcm_handle, params, channel_count);
+  snd_pcm_hw_params_set_buffer_size(pcm_handle, params, buffer_size);
+  snd_pcm_hw_params_set_format(pcm_handle, params, SND_PCM_FORMAT_S16_LE);
+  snd_pcm_hw_params_set_rate(pcm_handle, params, samples_per_second, 0);
+  snd_pcm_hw_params_set_periods(pcm_handle, params, period_size, 0);
+
+  err = snd_pcm_hw_params(pcm_handle, params);
+  if (err < 0) {
+    fprintf(stderr, "unable to prepare pcm\n");
+    snd_pcm_close(pcm_handle);
+    snd_pcm_hw_params_free(params);
+    return NULL;
+  }
+
+  print_pcm_state(pcm_handle);
+
+  err = snd_pcm_start(pcm_handle);
+  if (err < 0) {
+    fprintf(stderr, "unable to start pcm");
+    snd_pcm_close(pcm_handle);
+    snd_pcm_hw_params_free(params);
+    return NULL;
+  }
+
+  print_pcm_state(pcm_handle);
+
+  const uint seconds_to_capture = seconds;
+  const uint periods_to_read =
+      (samples_per_second * seconds_to_capture) / period_size;
+  char *buf = malloc(period_size * channel_count * bytes_per_sample);
+  uint periods_read = 0;
+  /* Read one period at a time from the hardware buffer
+   *  We set the hardware buffer to read 4096 frames.
+   *  The period size is 1024 frames.
+   *  Each period is 2048 BYTES because each frame is 2 bytes.
+   *  Each frame is 2 bytes because of 1 channel, and 16 bit format
+   */
+  FILE *file = fopen(output_file, "wb");
+  if (file == NULL) {
+    perror("Error opening file");
+    return NULL;
+  }
+  printf("======READING FRAMES for 5 seconds========\n");
+  while (snd_pcm_readi(pcm_handle, (void *)buf, period_size) > 0 &&
+         periods_read < periods_to_read) {
+    // Because each frame is 2 bytes (format/bit rate), cast to int16
+    // int16_t *samples = (int16_t *)buf;
+    // for (int i = 0; i < period_size; i++) {
+    //     printf("%d ", samples[i]);
+    // }
+
+    fwrite(buf, sizeof(int16_t), period_size, file);
+    periods_read++;
+  }
+  printf("done capturing audio\n");
+
+  fclose(file);
+
+  free(buf);
+  snd_pcm_close(pcm_handle);
+  snd_pcm_hw_params_free(params);
+
+  return output_file;
+}

ai-conversation/compile_commands.json

@@ -0,0 +1,16 @@
+[
+  {
+    "arguments": [
+      "/usr/bin/gcc",
+      "-c",
+      "-Wall",
+      "-g",
+      "-o",
+      "audio_capture",
+      "audio_capture.c"
+    ],
+    "directory": "/home/talksik/Documents/research_and_development/ai-conversation",
+    "file": "/home/talksik/Documents/research_and_development/ai-conversation/audio_capture.c",
+    "output": "/home/talksik/Documents/research_and_development/ai-conversation/audio_capture"
+  }
+]

ai-conversation/deepgram_client.c

@@ -0,0 +1,168 @@
+#include "deepgram_client.h"
+#include <string.h>
+
+// Returns string that caller must free
+static char *dg_find_transcript(char *json_response) {
+  printf("Parsing response: %s \n", json_response);
+  char *start = strstr(json_response, "\"transcript\":\"");
+  if (!start)
+    return NULL;
+
+  start += 14;                    // Skip past "transcript":"
+  char *end = strchr(start, '"'); // Find closing quote
+  // Copy substring between start and end
+  size_t len = end - start;
+  char *result = malloc(len + 1);
+  strncpy(result, start, len);
+  result[len] = '\0';
+  return result;
+}
+
+char *dg_transcribe(FILE *audio_file) {
+  CURL *curl;
+  CURLcode res;
+  struct curl_slist *headers = NULL;
+
+  // Initialize response buffer
+  struct curl_response_data response = {0};
+  response.data = malloc(1);
+  response.size = 0;
+
+  // Get file size
+  fseek(audio_file, 0, SEEK_END);
+  long file_size = ftell(audio_file);
+  fseek(audio_file, 0, SEEK_SET);
+
+  struct curl_file_data data;
+  data.file = audio_file;
+  data.size = file_size;
+
+  curl = curl_easy_init();
+  if (curl) {
+    // Set headers for Deepgram API
+    char *deepgram_api_key = getenv("DEEPGRAM_API_KEY");
+    if (!deepgram_api_key) {
+      fprintf(stderr, "DEEPGRAM_API_KEY environment variable not set\n");
+      free(response.data);
+      curl_easy_cleanup(curl);
+      return NULL;
+    }
+
+    char auth_header[256];
+    snprintf(auth_header, sizeof(auth_header), "Authorization: Token %s", deepgram_api_key);
+    headers = curl_slist_append(headers, auth_header);
+
+    curl_easy_setopt(curl, CURLOPT_URL,
+                     "https://api.deepgram.com/v1/"
+                     "listen?encoding=linear16&sample_rate=48000&channels=1");
+    curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
+    curl_easy_setopt(curl, CURLOPT_POST, 1L);
+    curl_easy_setopt(curl, CURLOPT_READFUNCTION, curl_read_callback_file);
+    curl_easy_setopt(curl, CURLOPT_READDATA, &data);
+    curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, file_size);
+    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curl_write_callback_response);
+    curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response);
+
+    res = curl_easy_perform(curl);
+    if (res != CURLE_OK) {
+      fprintf(stderr, "curl failed: %s\n", curl_easy_strerror(res));
+      free(response.data);
+      curl_slist_free_all(headers);
+      curl_easy_cleanup(curl);
+      return NULL;
+    }
+
+    printf("done performing curl\n");
+
+    curl_slist_free_all(headers);
+    curl_easy_cleanup(curl);
+  }
+
+  // Parse transcript from complete response
+  char *transcript = dg_find_transcript(response.data);
+  free(response.data);
+
+  return transcript;
+}
+
+struct curl_response_data *dg_text_to_speech(char *text) {
+  CURL *curl;
+  CURLcode res;
+  struct curl_slist *headers = NULL;
+
+  // Create JSON payload - need to escape the text properly
+  // Worst case: every char needs escaping, so allocate 2x + overhead
+  size_t text_len = strlen(text);
+  size_t payload_size = text_len * 2 + 100;
+  char *json_payload = malloc(payload_size);
+
+  // Simple approach: replace problematic chars with spaces
+  char *escaped_text = malloc(text_len + 1);
+  strcpy(escaped_text, text);
+  for (int i = 0; escaped_text[i]; i++) {
+    if (escaped_text[i] == '"' || escaped_text[i] == '\n' ||
+        escaped_text[i] == '\r' || escaped_text[i] == '\t') {
+      escaped_text[i] = ' ';
+    }
+  }
+
+  snprintf(json_payload, payload_size,
+           "{"
+           "\"text\":\"%s\""
+           "}",
+           escaped_text);
+
+  free(escaped_text);
+
+  // Initialize response buffer
+  struct curl_response_data *response =
+      malloc(sizeof(struct curl_response_data));
+  response->data = malloc(1);
+  response->size = 0;
+
+  curl = curl_easy_init();
+  if (curl) {
+    char *deepgram_api_key = getenv("DEEPGRAM_API_KEY");
+    if (!deepgram_api_key) {
+      fprintf(stderr, "DEEPGRAM_API_KEY environment variable not set\n");
+      free(response->data);
+      curl_easy_cleanup(curl);
+      return NULL;
+    }
+
+    char auth_header[256];
+    snprintf(auth_header, sizeof(auth_header), "Authorization: Token %s", deepgram_api_key);
+    headers = curl_slist_append(headers, auth_header);
+    headers = curl_slist_append(headers, "Content-Type: application/json");
+
+    curl_easy_setopt(curl, CURLOPT_URL,
+                     "https://api.deepgram.com/v1/"
+                     "speak?encoding=linear16&sample_rate=48000");
+    curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
+    curl_easy_setopt(curl, CURLOPT_POST, 1L);
+    curl_easy_setopt(curl, CURLOPT_POSTFIELDS, json_payload);
+    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curl_write_callback_response);
+    curl_easy_setopt(curl, CURLOPT_WRITEDATA, response);
+
+    res = curl_easy_perform(curl);
+    if (res != CURLE_OK) {
+      fprintf(stderr, "curl failed: %s\n", curl_easy_strerror(res));
+      free(response->data);
+      free(response);
+      curl_slist_free_all(headers);
+      curl_easy_cleanup(curl);
+      return NULL;
+    }
+
+    curl_slist_free_all(headers);
+    curl_easy_cleanup(curl);
+  }
+
+  printf("received audio of %d length for tts\n", (int)response->size);
+
+  // Debug: print first 100 chars to see if it's audio or error message
+  printf("First 100 chars: %.100s\n", response->data);
+
+  free(json_payload);
+  return response;
+}

ai-conversation/include/audio.h

@@ -0,0 +1,29 @@
+#ifndef AUDIOCAPTURE_H
+#define AUDIOCAPTURE_H
+
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <alsa/asoundlib.h>
+#include <alsa/control.h>
+#include <alsa/pcm.h>
+
+// NOTE: find this via `arecord -l`
+#define ALSA_MIC_HARDWARE_DEVICE "hw:3"
+#define ALSA_SPEAKER_HARDWARE_DEVICE "default"
+
+// Captures audio for given seconds, and returns file name that we captured the
+// audio into
+char *audio_capture(uint seconds);
+
+/* count: how many bytes in the buffer
+ *  returns int: 0 if success, -1 if failure
+ *  NOTE: This function assumes that the audio data is using:
+ *  - linear16 formatting (each sample is 16 bits)
+ *  - one channel (1 sample)
+ *  - with 48000 HZ sampling rate
+ *  Hence, it's 2 bytes per frame.
+ */
+int audio_play(char *buf, size_t byte_count);
+
+#endif // AUDIOCAPTURE_H

ai-conversation/include/deepgram_client.h

@@ -0,0 +1,21 @@
+#ifndef DEEPGRAMCLIENT_H
+#define DEEPGRAMCLIENT_H
+
+#include <curl/curl.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include "utils.h"
+
+/*
+ * @param audio_file: must already be opened
+ * @returns char *: transcript string result. Call must free.
+ */
+char *dg_transcribe(FILE *audio_file);
+
+/*
+*  @returns response data. Caller must free the struct AND the data within it.
+*  TODO: should return custom struct with details about encoding, sample rate, etc.
+*/
+struct curl_response_data *dg_text_to_speech(char *text);
+
+#endif // DEEPGRAMCLIENT_H

ai-conversation/include/intelligence.h

@@ -0,0 +1,9 @@
+#ifndef INTELLIGENCE_H
+#define INTELLIGENCE_H
+
+#include <stdio.h>
+#include <stdlib.h>
+
+char *ai_ask(char *question);
+
+#endif //INTELLIGENCE_H

ai-conversation/include/utils.h

@@ -0,0 +1,24 @@
+#ifndef UTILS_H
+#define UTILS_H
+
+#include <stdlib.h>
+#include <stdio.h>
+
+struct curl_file_data {
+  FILE *file;
+  size_t size;
+};
+
+struct curl_response_data {
+  char *data;
+  size_t size;
+};
+
+// This callback assumes the userdata is `struct file_data`
+size_t curl_read_callback_file(void *ptr, size_t size, size_t nmemb, void *userdata);
+
+// This callback assumes that we are aggregating callback data into an allocated response_data struct
+size_t curl_write_callback_response(void *contents, size_t size, size_t nmemb,
+                             void *userp);
+
+#endif // UTILS_H

ai-conversation/intelligence.c

@@ -0,0 +1,86 @@
+#include "intelligence.h"
+#include "utils.h"
+#include <curl/curl.h>
+#include <string.h>
+
+static char *find_ai_response(char *json_response) {
+  printf("Parsing response: %s \n", json_response);
+  char *start = strstr(json_response, "\"text\":\"");
+  if (!start)
+    return NULL;
+
+  start += 8; // Skip past "text":"
+  char *end = strchr(start, '"');
+  if (!end)
+    return NULL;
+
+  size_t len = end - start;
+  char *result = malloc(len + 1);
+  strncpy(result, start, len);
+  result[len] = '\0';
+  return result;
+}
+
+char *ai_ask(char *question) {
+  CURL *curl;
+  CURLcode res;
+  struct curl_slist *headers = NULL;
+
+  // Initialize response buffer
+  struct curl_response_data response = {0};
+  response.data = malloc(1);
+  response.size = 0;
+
+  // Create JSON payload
+  char json_payload[2048];
+  snprintf(json_payload, sizeof(json_payload),
+           "{"
+           "\"model\":\"claude-3-haiku-20240307\","
+           "\"max_tokens\":256,"
+           "\"messages\":[{\"role\":\"user\",\"content\":\"Make it one sentence answer: %s\"}]"
+           "}",
+           question);
+
+  curl = curl_easy_init();
+  if (curl) {
+    char *anthropic_api_key = getenv("ANTHROPIC_API_KEY");
+    if (!anthropic_api_key) {
+      printf("ANTHROPIC_API_KEY not set\n");
+      free(response.data);
+      curl_easy_cleanup(curl);
+      return NULL;
+    }
+    char auth_header[256];
+    snprintf(auth_header, sizeof(auth_header), "x-api-key: %s",
+             anthropic_api_key);
+    headers = curl_slist_append(headers, "content-type: application/json");
+    headers = curl_slist_append(headers, auth_header);
+    headers = curl_slist_append(headers, "anthropic-version: 2023-06-01");
+
+    curl_easy_setopt(curl, CURLOPT_URL,
+                     "https://api.anthropic.com/v1/messages");
+    curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
+    curl_easy_setopt(curl, CURLOPT_POST, 1L);
+    curl_easy_setopt(curl, CURLOPT_POSTFIELDS, json_payload);
+    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curl_write_callback_response);
+    curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response);
+
+    res = curl_easy_perform(curl);
+    if (res != CURLE_OK) {
+      fprintf(stderr, "curl failed: %s\n", curl_easy_strerror(res));
+      free(response.data);
+      curl_slist_free_all(headers);
+      curl_easy_cleanup(curl);
+      return NULL;
+    }
+
+    curl_slist_free_all(headers);
+    curl_easy_cleanup(curl);
+  }
+
+  // Parse AI response from complete response
+  char *answer = find_ai_response(response.data);
+  free(response.data);
+
+  return answer;
+}

ai-conversation/main.c

@@ -0,0 +1,45 @@
+#include "audio.h"
+#include "deepgram_client.h"
+#include "intelligence.h"
+#include "utils.h"
+#include <stdio.h>
+#include <stdlib.h>
+
+int main(int argc, char *argv[]) {
+  char *capture_audio_file = audio_capture(5);
+
+  // Reopen file for reading and transcribe
+  FILE *file = fopen(capture_audio_file, "rb");
+  if (file != NULL) {
+    char *transcript = dg_transcribe(file);
+    if (transcript) {
+      printf("Transcript: %s\n", transcript);
+
+      char *ai_answer = ai_ask(transcript);
+      if (ai_answer) {
+        printf("AI answer: %s\n", ai_answer);
+
+        struct curl_response_data *tts_response = dg_text_to_speech(ai_answer);
+        int result = audio_play(tts_response->data, tts_response->size);
+        if (result != 0) {
+          fprintf(stderr, "failure in playback");
+        }
+
+        free(tts_response->data);
+        free(tts_response);
+
+        // we have this response data, and buffer within it. we can loop through
+        // it and write the pcm data within it to alsa
+        free(ai_answer);
+      }
+      free(transcript);
+    } else {
+      fprintf(stderr, "unable to get transcript");
+    }
+    fclose(file);
+  } else {
+    fprintf(stderr, "unable to open file for reading\n");
+  }
+
+  return 0;
+}

ai-conversation/utils.c

@@ -0,0 +1,30 @@
+#include "utils.h"
+#include <string.h>
+
+size_t curl_read_callback_file(void *ptr, size_t size, size_t nmemb,
+                            void *userdata) {
+  struct curl_file_data *data = (struct curl_file_data *)userdata;
+  size_t bytes_to_read = size * nmemb;
+  return fread(ptr, 1, bytes_to_read, data->file);
+}
+
+size_t curl_write_callback_response(void *contents, size_t size, size_t nmemb,
+                             void *userp) {
+  size_t total_size = size * nmemb;
+  struct curl_response_data *response = (struct curl_response_data *)userp;
+
+  // Reallocate buffer to fit new data
+  response->data = realloc(response->data, response->size + total_size + 1);
+  if (response->data == NULL) {
+    return 0; // Error
+  }
+
+  // Copy new data to buffer
+  memcpy(&response->data[response->size], contents, total_size);
+  response->size += total_size;
+  response->data[response->size] = '\0'; // Null terminate
+
+  printf("read an extra %d bytes of data\n", (int)total_size);
+
+  return total_size;
+}

stream/.gitignore

@@ -1,4 +0,0 @@
-.cache
-stream_app
-*.gguf
-compile_commands.json

stream/Makefile

@@ -1,11 +0,0 @@
-CC = gcc
-CFLAGS = -Wall -g -I./thirdparty/llama.cpp/include -I./thirdparty/llama.cpp/ggml/include
-TARGET = stream_app
-SRC = main.c
-LIBS = -L./thirdparty/llama.cpp/build/bin -lllama -lggml -lggml-base
-
-$(TARGET): $(SRC)
-	$(CC) $(CFLAGS) -o $(TARGET) $(SRC) $(LIBS)
-
-clean:
-	rm -f *.o $(TARGET)

stream/README.md

@@ -1,38 +0,0 @@
-# flowy.stream
-
-This is a proof of concept of a showing widgets within a conversation based on intent.
-
-## Todo
-- [x] Integrate llama.cpp with local inference. This will set us up for building many parts of experience.
-- [ ] Disect what llama is doing and what the Phi model is doing.
-- [ ] Create conversational loop with chat and running context.
-- [ ] Generate & render different types of blocks: list, email, doc, etc.
-- [ ] Try different models with hugging face
-- [ ] Render TUI elements based on commands
-
-## Dependencies
-
-### llama.cpp
-```sh
-#model weights
-wget https://huggingface.co/microsoft/Phi-3-mini-4k-instruct-gguf/resolve/main/Phi-3-mini-4k-instruct-q4.gguf
-```
-
-```sh
-cd ./thirdparty/llama.cpp
-
-rm -rf build
-mkdir -p build
-cmake -S. -Bbuild -DLLAMA_CURL=OFF
-cmake --build build
-```
-
-The libraries should be in `./thirdparty/llama.cpp/build/bin`.
-
-NOTE: you may have to disable curl as a flag when configuring llama.cpp via cmake.
-
-## Getting started
-```sh
-make
-LD_LIBRARY_PATH=./thirdparty/llama.cpp/build/bin/ ./stream_app
-```

stream/main.c

@@ -1,176 +0,0 @@
-#include "thirdparty/llama.cpp/ggml/include/ggml-backend.h"
-#include <llama.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
-typedef enum {
-  Unspecified,
-  Question,
-  Math,
-  Code,
-  Email,
-  Doc,
-  WebSearch,
-} IntentType;
-
-int main(int argc, char *argv[]) {
-  printf("Hello world\n");
-
-  char user_input[500];
-  printf("What's up, what can I help with?\n");
-  fgets(user_input, sizeof(user_input), stdin);
-
-  char prompt[1000];
-  snprintf(prompt, sizeof(prompt), "<|system|>You are a helpful assistant. Your goal is to take in what the user is doing and return 3 predictive actions/3 suggestions based on what the user is trying to do: e.g. change page title when in google sheets, or calculate sum, create chart.<|end|>\n<|user|>%s<|end|>\n<|assistant|>", user_input);
-
-  // number of layers to offload to the GPU
-  int ngl = 99;
-  // number of tokens to predict
-  int n_predict = 1000;
-
-  // load dynamic backends
-
-  ggml_backend_load_all();
-
-  // initialize the model
-
-  struct llama_model_params model_params = llama_model_default_params();
-  model_params.n_gpu_layers = ngl;
-
-  struct llama_model *model = llama_model_load_from_file(
-      "Phi-3-mini-4k-instruct-q4.gguf", model_params);
-
-  if (model == NULL) {
-    fprintf(stderr, "%s: error: unable to load model\n", __func__);
-    return 1;
-  }
-
-  const struct llama_vocab *vocab = llama_model_get_vocab(model);
-  // tokenize the prompt
-
-  // find the number of tokens in the prompt
-  const int n_prompt =
-      -llama_tokenize(vocab, prompt, strlen(prompt), NULL, 0, true, true);
-
-  // allocate space for the tokens and tokenize the prompt
-  llama_token *prompt_tokens = malloc(n_prompt * sizeof(llama_token));
-  if (prompt_tokens == NULL) {
-    fprintf(stderr, "%s: error: failed to allocate memory for prompt tokens\n",
-            __func__);
-    return 1;
-  }
-  if (llama_tokenize(vocab, prompt, strlen(prompt), prompt_tokens, n_prompt,
-                     true, true) < 0) {
-    fprintf(stderr, "%s: error: failed to tokenize the prompt\n", __func__);
-    return 1;
-  }
-
-  // initialize the context
-
-  struct llama_context_params ctx_params = llama_context_default_params();
-  // n_ctx is the context size
-  ctx_params.n_ctx = n_prompt + n_predict - 1;
-  // n_batch is the maximum number of tokens that can be processed in a single
-  // call to llama_decode
-  ctx_params.n_batch = n_prompt;
-  // enable performance counters
-  ctx_params.no_perf = false;
-
-  struct llama_context *ctx = llama_init_from_model(model, ctx_params);
-
-  if (ctx == NULL) {
-    fprintf(stderr, "%s: error: failed to create the llama_context\n",
-            __func__);
-    return 1;
-  }
-
-  // initialize the sampler
-
-  struct llama_sampler_chain_params sparams =
-      llama_sampler_chain_default_params();
-  sparams.no_perf = false;
-  struct llama_sampler *smpl = llama_sampler_chain_init(sparams);
-
-  llama_sampler_chain_add(smpl, llama_sampler_init_greedy());
-
-  // print the prompt token-by-token
-
-  for (int i = 0; i < n_prompt; i++) {
-    char buf[128];
-    int n = llama_token_to_piece(vocab, prompt_tokens[i], buf, sizeof(buf), 0,
-                                 true);
-    if (n < 0) {
-      fprintf(stderr, "%s: error: failed to convert token to piece\n",
-              __func__);
-      return 1;
-    }
-    buf[n] = '\0';
-    printf("%s", buf);
-  }
-
-  // prepare a batch for the prompt
-
-  llama_batch batch = llama_batch_get_one(prompt_tokens, n_prompt);
-
-  // main loop
-
-  const int64_t t_main_start = ggml_time_us();
-  int n_decode = 0;
-  llama_token new_token_id;
-
-  for (int n_pos = 0; n_pos + batch.n_tokens < n_prompt + n_predict;) {
-    // evaluate the current batch with the transformer model
-    if (llama_decode(ctx, batch)) {
-      fprintf(stderr, "%s : failed to eval, return code %d\n", __func__, 1);
-      return 1;
-    }
-
-    n_pos += batch.n_tokens;
-
-    // sample the next token
-    {
-      new_token_id = llama_sampler_sample(smpl, ctx, -1);
-
-      // is it an end of generation?
-      if (llama_vocab_is_eog(vocab, new_token_id)) {
-        break;
-      }
-
-      char buf[128];
-      int n =
-          llama_token_to_piece(vocab, new_token_id, buf, sizeof(buf), 0, true);
-      if (n < 0) {
-        fprintf(stderr, "%s: error: failed to convert token to piece\n",
-                __func__);
-        return 1;
-      }
-      buf[n] = '\0';
-      printf("%s", buf);
-      fflush(stdout);
-
-      // prepare the next batch with the sampled token
-      batch = llama_batch_get_one(&new_token_id, 1);
-
-      n_decode += 1;
-    }
-  }
-
-  printf("\n");
-
-  const int64_t t_main_end = ggml_time_us();
-
-  fprintf(stderr, "%s: decoded %d tokens in %.2f s, speed: %.2f t/s\n",
-          __func__, n_decode, (t_main_end - t_main_start) / 1000000.0f,
-          n_decode / ((t_main_end - t_main_start) / 1000000.0f));
-
-  fprintf(stderr, "\n");
-  llama_perf_sampler_print(smpl);
-  llama_perf_context_print(ctx);
-  fprintf(stderr, "\n");
-
-  llama_sampler_free(smpl);
-  llama_free(ctx);
-  llama_model_free(model);
-  free(prompt_tokens);
-}