14 Commits
Author SHA1 Message Date
talksik 70a36a329c docs: add disclaimer about hardcoded audio params 2025-07-27 15:03:20 -07:00
talksik 12a8543564 docs: explain future work 2025-07-27 15:00:16 -07:00
talksik 060bbfb2c4 docs: add readme 2025-07-27 14:58:00 -07:00
talksik 8c1596aad8 organize header files into include directory 2025-07-27 14:51:42 -07:00
talksik 9ca1f8273c organize audio and intelligence modules 2025-07-27 14:47:41 -07:00
talksik c93267c9ee create full flow with stt, anthropic, tts, and playback
This organizes some components like deepgram into own module, and also
allows gets the full flow to work every time we run the program.
2025-07-27 14:18:12 -07:00
talksik 1e4211052e hit anthropic API for ai question 2025-07-26 21:34:11 -07:00
talksik 966050a7b4 transcribe raw audio 2025-07-26 19:30:03 -07:00
talksik ed3850abd6 transcribe audio file with deepgram 2025-07-26 18:58:21 -07:00
talksik cd8ea7c5ed write pcm data to file 2025-07-26 18:32:48 -07:00
talksik 1d0cb5bb93 add libcurl basic example 2025-07-26 18:30:12 -07:00
talksik 3bb287fcd5 use basecamp project for managing this project 2025-07-26 17:24:56 -07:00
talksik 6af3f71510 add brainstorm thoughts 2025-07-26 17:15:14 -07:00
talksik 5abbeb5dcd capture audio with alsa 2025-07-13 11:52:09 -07:00
15 changed files with 693 additions and 0 deletions
+2
View File
@@ -0,0 +1,2 @@
main
.cache
+14
View File
@@ -0,0 +1,14 @@
CC = gcc
CFLAGS = -Wall -g -Iinclude
TARGET = main
SRC = main.c deepgram_client.c utils.c intelligence.c audio.c
LIBS = -lavdevice -lavformat -lavcodec -lavutil -lasound -lcurl
$(TARGET): $(SRC)
$(CC) $(CFLAGS) -o $(TARGET) $(SRC) $(LIBS)
clean:
rm -f *.o $(TARGET)
install-deps:
sudo apt install build-essential libavformat-dev libavdevice-dev libavcodec-dev libavutil-dev libasound2-dev
+34
View File
@@ -0,0 +1,34 @@
# Conversation
This proof of concept is for implementing conversational loop with AI at a low level with raw C.
## How it works
This simple program:
1. Captures audio samples for x seconds into a local file using alsa
2. Sends the raw pcm data to Deepgram STT API for transcription
3. Uses the transcript to call Anthropic to get an assistant's response
4. Converts Anthropic's response to audio using Deepgram's TTS
5. Plays back the generated audio from Deepgram using alsa
6. Quits. _There is no conversational loop at the time of this writing._
## Requirements
Platform: the project is meant for Linux x64. It was successfully run on arch linux, with no guarantees otherwise.
* libcurl
* build-essential
* libasound2-dev
_See more deps in Makefile `install-deps` command._
## Getting started
1. Add `DEEPGRAM_API_KEY` and `ANTHROPIC_API_KEY` to `.bashrc` as environment variables.
2. Build & run
```sh
make
./main
```
## Audio configuration
Please see the `audio.h` and `audio.c` as the params for alsa were hardcoded based on my local arch linux x64 machine. You may have to adjust it based on your setup (e.g. you may have a mic with 2 channels/stereo, which would change the size of each frame).
## Future work
Add an event loop, maybe add multi-threading, so that we can interrupt the AI, ask follow-up questions, and make it sound like a back and forth conversation.
+215
View File
@@ -0,0 +1,215 @@
#include "audio.h"
static void print_pcm_state(snd_pcm_t *pcm_handle) {
snd_pcm_state_t state = snd_pcm_state(pcm_handle);
const char *state_name = snd_pcm_state_name(state);
printf("State of pcm handle: %s\n", state_name);
}
int audio_play(char *buf, size_t byte_count) {
printf("going to play audio of %d bytes of data\n", (int)byte_count);
snd_pcm_t *pcm_handle;
// The total frames in the buffer is how many bytes divided by 2 because
// of linear16 encoding, one channel
size_t total_frames = byte_count / 2;
int err;
err = snd_pcm_open(&pcm_handle, ALSA_SPEAKER_HARDWARE_DEVICE,
SND_PCM_STREAM_PLAYBACK, 0);
if (err < 0) {
fprintf(stderr, "unable to open playback device");
return err;
}
print_pcm_state(pcm_handle);
const char *name = snd_pcm_name(pcm_handle);
printf("found this device: %s\n", name);
snd_pcm_info_t *pcm_info;
snd_pcm_info_malloc(&pcm_info);
err = snd_pcm_info(pcm_handle, pcm_info);
if (err < 0) {
snd_pcm_close(pcm_handle);
printf("unable to get info of pcm device");
return -1;
}
name = snd_pcm_info_get_name(pcm_info);
if (err < 0) {
printf("unable to get card info");
snd_pcm_close(pcm_handle);
snd_pcm_info_free(pcm_info);
return -1;
}
printf("got pcm name: %s \n", name);
print_pcm_state(pcm_handle);
snd_pcm_info_free(pcm_info);
if ((err = snd_pcm_set_params(pcm_handle, SND_PCM_FORMAT_S16_LE,
SND_PCM_ACCESS_RW_INTERLEAVED, 1, 48000, 1,
500000)) < 0) { /* 0.5sec */
printf("Playback open error: %s\n", snd_strerror(err));
exit(EXIT_FAILURE);
}
// we want to read in a chunk of data and writei to the pcm
size_t frames_written = 0;
size_t frames_in_each_chunk = 1024; // how many frames to read per chunk (2048
// bytes since each frame is 2 bytes)
while (frames_written < total_frames) {
size_t frames_to_write =
frames_in_each_chunk < (total_frames - frames_written)
? frames_in_each_chunk
: (total_frames - frames_written);
// We move the pointer x2 because each frame is 2 bytes so the next audio
// sample is 16 bits forward, not 8
char *start_ptr = buf + (frames_written * 2);
snd_pcm_sframes_t result =
snd_pcm_writei(pcm_handle, start_ptr, frames_to_write);
if (result < 0) {
result = snd_pcm_recover(pcm_handle, result, 0);
if (result < 0) {
printf("snd_pcm_writei failed: %s\n", snd_strerror(result));
break;
}
}
if (result > 0 && result < frames_to_write) {
printf("Short write \n");
}
frames_written += result;
}
/* pass the remaining samples, otherwise they're dropped in close */
err = snd_pcm_drain(pcm_handle);
if (err < 0) {
printf("snd_pcm_drain failed: %s\n", snd_strerror(err));
}
snd_pcm_close(pcm_handle);
return 0;
}
char *audio_capture(uint seconds) {
char *output_file = "output.pcm";
snd_pcm_t *pcm_handle = NULL;
int err;
err = snd_pcm_open(&pcm_handle, ALSA_MIC_HARDWARE_DEVICE,
SND_PCM_STREAM_CAPTURE, 0);
if (err < 0) {
printf("unable to open pcm device");
return NULL;
}
print_pcm_state(pcm_handle);
const char *name = snd_pcm_name(pcm_handle);
printf("found this device: %s\n", name);
snd_pcm_info_t *pcm_info;
snd_pcm_info_malloc(&pcm_info);
err = snd_pcm_info(pcm_handle, pcm_info);
if (err < 0) {
snd_pcm_close(pcm_handle);
printf("unable to get info of pcm device");
return NULL;
}
name = snd_pcm_info_get_name(pcm_info);
if (err < 0) {
printf("unable to get card info");
snd_pcm_close(pcm_handle);
snd_pcm_info_free(pcm_info);
return NULL;
}
printf("got pcm name: %s \n", name);
print_pcm_state(pcm_handle);
snd_pcm_info_free(pcm_info);
snd_pcm_hw_params_t *params;
err = snd_pcm_hw_params_malloc(&params);
if (err < 0) {
fprintf(stderr, "unable to mallow hw params");
return NULL;
}
err = snd_pcm_hw_params_any(pcm_handle, params);
if (err < 0) {
fprintf(stderr, "unable to find ranges for hw device");
snd_pcm_close(pcm_handle);
snd_pcm_hw_params_free(params);
return NULL;
}
uint period_size = 1024;
uint buffer_size = 4096;
uint channel_count = 1;
uint bytes_per_sample = 2; // FORMAT_S16_LE
uint samples_per_second = 48000;
snd_pcm_hw_params_set_access(pcm_handle, params,
SND_PCM_ACCESS_RW_INTERLEAVED);
snd_pcm_hw_params_set_channels(pcm_handle, params, channel_count);
snd_pcm_hw_params_set_buffer_size(pcm_handle, params, buffer_size);
snd_pcm_hw_params_set_format(pcm_handle, params, SND_PCM_FORMAT_S16_LE);
snd_pcm_hw_params_set_rate(pcm_handle, params, samples_per_second, 0);
snd_pcm_hw_params_set_periods(pcm_handle, params, period_size, 0);
err = snd_pcm_hw_params(pcm_handle, params);
if (err < 0) {
fprintf(stderr, "unable to prepare pcm\n");
snd_pcm_close(pcm_handle);
snd_pcm_hw_params_free(params);
return NULL;
}
print_pcm_state(pcm_handle);
err = snd_pcm_start(pcm_handle);
if (err < 0) {
fprintf(stderr, "unable to start pcm");
snd_pcm_close(pcm_handle);
snd_pcm_hw_params_free(params);
return NULL;
}
print_pcm_state(pcm_handle);
const uint seconds_to_capture = seconds;
const uint periods_to_read =
(samples_per_second * seconds_to_capture) / period_size;
char *buf = malloc(period_size * channel_count * bytes_per_sample);
uint periods_read = 0;
/* Read one period at a time from the hardware buffer
* We set the hardware buffer to read 4096 frames.
* The period size is 1024 frames.
* Each period is 2048 BYTES because each frame is 2 bytes.
* Each frame is 2 bytes because of 1 channel, and 16 bit format
*/
FILE *file = fopen(output_file, "wb");
if (file == NULL) {
perror("Error opening file");
return NULL;
}
printf("======READING FRAMES for 5 seconds========\n");
while (snd_pcm_readi(pcm_handle, (void *)buf, period_size) > 0 &&
periods_read < periods_to_read) {
// Because each frame is 2 bytes (format/bit rate), cast to int16
// int16_t *samples = (int16_t *)buf;
// for (int i = 0; i < period_size; i++) {
// printf("%d ", samples[i]);
// }
fwrite(buf, sizeof(int16_t), period_size, file);
periods_read++;
}
printf("done capturing audio\n");
fclose(file);
free(buf);
snd_pcm_close(pcm_handle);
snd_pcm_hw_params_free(params);
return output_file;
}
+16
View File
@@ -0,0 +1,16 @@
[
{
"arguments": [
"/usr/bin/gcc",
"-c",
"-Wall",
"-g",
"-o",
"audio_capture",
"audio_capture.c"
],
"directory": "/home/talksik/Documents/research_and_development/ai-conversation",
"file": "/home/talksik/Documents/research_and_development/ai-conversation/audio_capture.c",
"output": "/home/talksik/Documents/research_and_development/ai-conversation/audio_capture"
}
]
+168
View File
@@ -0,0 +1,168 @@
#include "deepgram_client.h"
#include <string.h>
// Returns string that caller must free
static char *dg_find_transcript(char *json_response) {
printf("Parsing response: %s \n", json_response);
char *start = strstr(json_response, "\"transcript\":\"");
if (!start)
return NULL;
start += 14; // Skip past "transcript":"
char *end = strchr(start, '"'); // Find closing quote
// Copy substring between start and end
size_t len = end - start;
char *result = malloc(len + 1);
strncpy(result, start, len);
result[len] = '\0';
return result;
}
char *dg_transcribe(FILE *audio_file) {
CURL *curl;
CURLcode res;
struct curl_slist *headers = NULL;
// Initialize response buffer
struct curl_response_data response = {0};
response.data = malloc(1);
response.size = 0;
// Get file size
fseek(audio_file, 0, SEEK_END);
long file_size = ftell(audio_file);
fseek(audio_file, 0, SEEK_SET);
struct curl_file_data data;
data.file = audio_file;
data.size = file_size;
curl = curl_easy_init();
if (curl) {
// Set headers for Deepgram API
char *deepgram_api_key = getenv("DEEPGRAM_API_KEY");
if (!deepgram_api_key) {
fprintf(stderr, "DEEPGRAM_API_KEY environment variable not set\n");
free(response.data);
curl_easy_cleanup(curl);
return NULL;
}
char auth_header[256];
snprintf(auth_header, sizeof(auth_header), "Authorization: Token %s", deepgram_api_key);
headers = curl_slist_append(headers, auth_header);
curl_easy_setopt(curl, CURLOPT_URL,
"https://api.deepgram.com/v1/"
"listen?encoding=linear16&sample_rate=48000&channels=1");
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
curl_easy_setopt(curl, CURLOPT_POST, 1L);
curl_easy_setopt(curl, CURLOPT_READFUNCTION, curl_read_callback_file);
curl_easy_setopt(curl, CURLOPT_READDATA, &data);
curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, file_size);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curl_write_callback_response);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response);
res = curl_easy_perform(curl);
if (res != CURLE_OK) {
fprintf(stderr, "curl failed: %s\n", curl_easy_strerror(res));
free(response.data);
curl_slist_free_all(headers);
curl_easy_cleanup(curl);
return NULL;
}
printf("done performing curl\n");
curl_slist_free_all(headers);
curl_easy_cleanup(curl);
}
// Parse transcript from complete response
char *transcript = dg_find_transcript(response.data);
free(response.data);
return transcript;
}
struct curl_response_data *dg_text_to_speech(char *text) {
CURL *curl;
CURLcode res;
struct curl_slist *headers = NULL;
// Create JSON payload - need to escape the text properly
// Worst case: every char needs escaping, so allocate 2x + overhead
size_t text_len = strlen(text);
size_t payload_size = text_len * 2 + 100;
char *json_payload = malloc(payload_size);
// Simple approach: replace problematic chars with spaces
char *escaped_text = malloc(text_len + 1);
strcpy(escaped_text, text);
for (int i = 0; escaped_text[i]; i++) {
if (escaped_text[i] == '"' || escaped_text[i] == '\n' ||
escaped_text[i] == '\r' || escaped_text[i] == '\t') {
escaped_text[i] = ' ';
}
}
snprintf(json_payload, payload_size,
"{"
"\"text\":\"%s\""
"}",
escaped_text);
free(escaped_text);
// Initialize response buffer
struct curl_response_data *response =
malloc(sizeof(struct curl_response_data));
response->data = malloc(1);
response->size = 0;
curl = curl_easy_init();
if (curl) {
char *deepgram_api_key = getenv("DEEPGRAM_API_KEY");
if (!deepgram_api_key) {
fprintf(stderr, "DEEPGRAM_API_KEY environment variable not set\n");
free(response->data);
curl_easy_cleanup(curl);
return NULL;
}
char auth_header[256];
snprintf(auth_header, sizeof(auth_header), "Authorization: Token %s", deepgram_api_key);
headers = curl_slist_append(headers, auth_header);
headers = curl_slist_append(headers, "Content-Type: application/json");
curl_easy_setopt(curl, CURLOPT_URL,
"https://api.deepgram.com/v1/"
"speak?encoding=linear16&sample_rate=48000");
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
curl_easy_setopt(curl, CURLOPT_POST, 1L);
curl_easy_setopt(curl, CURLOPT_POSTFIELDS, json_payload);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curl_write_callback_response);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, response);
res = curl_easy_perform(curl);
if (res != CURLE_OK) {
fprintf(stderr, "curl failed: %s\n", curl_easy_strerror(res));
free(response->data);
free(response);
curl_slist_free_all(headers);
curl_easy_cleanup(curl);
return NULL;
}
curl_slist_free_all(headers);
curl_easy_cleanup(curl);
}
printf("received audio of %d length for tts\n", (int)response->size);
// Debug: print first 100 chars to see if it's audio or error message
printf("First 100 chars: %.100s\n", response->data);
free(json_payload);
return response;
}
+29
View File
@@ -0,0 +1,29 @@
#ifndef AUDIOCAPTURE_H
#define AUDIOCAPTURE_H
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <alsa/asoundlib.h>
#include <alsa/control.h>
#include <alsa/pcm.h>
// NOTE: find this via `arecord -l`
#define ALSA_MIC_HARDWARE_DEVICE "hw:3"
#define ALSA_SPEAKER_HARDWARE_DEVICE "default"
// Captures audio for given seconds, and returns file name that we captured the
// audio into
char *audio_capture(uint seconds);
/* count: how many bytes in the buffer
* returns int: 0 if success, -1 if failure
* NOTE: This function assumes that the audio data is using:
* - linear16 formatting (each sample is 16 bits)
* - one channel (1 sample)
* - with 48000 HZ sampling rate
* Hence, it's 2 bytes per frame.
*/
int audio_play(char *buf, size_t byte_count);
#endif // AUDIOCAPTURE_H
+21
View File
@@ -0,0 +1,21 @@
#ifndef DEEPGRAMCLIENT_H
#define DEEPGRAMCLIENT_H
#include <curl/curl.h>
#include <stdio.h>
#include <stdlib.h>
#include "utils.h"
/*
* @param audio_file: must already be opened
* @returns char *: transcript string result. Call must free.
*/
char *dg_transcribe(FILE *audio_file);
/*
* @returns response data. Caller must free the struct AND the data within it.
* TODO: should return custom struct with details about encoding, sample rate, etc.
*/
struct curl_response_data *dg_text_to_speech(char *text);
#endif // DEEPGRAMCLIENT_H
+9
View File
@@ -0,0 +1,9 @@
#ifndef INTELLIGENCE_H
#define INTELLIGENCE_H
#include <stdio.h>
#include <stdlib.h>
char *ai_ask(char *question);
#endif //INTELLIGENCE_H
+24
View File
@@ -0,0 +1,24 @@
#ifndef UTILS_H
#define UTILS_H
#include <stdlib.h>
#include <stdio.h>
struct curl_file_data {
FILE *file;
size_t size;
};
struct curl_response_data {
char *data;
size_t size;
};
// This callback assumes the userdata is `struct file_data`
size_t curl_read_callback_file(void *ptr, size_t size, size_t nmemb, void *userdata);
// This callback assumes that we are aggregating callback data into an allocated response_data struct
size_t curl_write_callback_response(void *contents, size_t size, size_t nmemb,
void *userp);
#endif // UTILS_H
+86
View File
@@ -0,0 +1,86 @@
#include "intelligence.h"
#include "utils.h"
#include <curl/curl.h>
#include <string.h>
static char *find_ai_response(char *json_response) {
printf("Parsing response: %s \n", json_response);
char *start = strstr(json_response, "\"text\":\"");
if (!start)
return NULL;
start += 8; // Skip past "text":"
char *end = strchr(start, '"');
if (!end)
return NULL;
size_t len = end - start;
char *result = malloc(len + 1);
strncpy(result, start, len);
result[len] = '\0';
return result;
}
char *ai_ask(char *question) {
CURL *curl;
CURLcode res;
struct curl_slist *headers = NULL;
// Initialize response buffer
struct curl_response_data response = {0};
response.data = malloc(1);
response.size = 0;
// Create JSON payload
char json_payload[2048];
snprintf(json_payload, sizeof(json_payload),
"{"
"\"model\":\"claude-3-haiku-20240307\","
"\"max_tokens\":256,"
"\"messages\":[{\"role\":\"user\",\"content\":\"Make it one sentence answer: %s\"}]"
"}",
question);
curl = curl_easy_init();
if (curl) {
char *anthropic_api_key = getenv("ANTHROPIC_API_KEY");
if (!anthropic_api_key) {
printf("ANTHROPIC_API_KEY not set\n");
free(response.data);
curl_easy_cleanup(curl);
return NULL;
}
char auth_header[256];
snprintf(auth_header, sizeof(auth_header), "x-api-key: %s",
anthropic_api_key);
headers = curl_slist_append(headers, "content-type: application/json");
headers = curl_slist_append(headers, auth_header);
headers = curl_slist_append(headers, "anthropic-version: 2023-06-01");
curl_easy_setopt(curl, CURLOPT_URL,
"https://api.anthropic.com/v1/messages");
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
curl_easy_setopt(curl, CURLOPT_POST, 1L);
curl_easy_setopt(curl, CURLOPT_POSTFIELDS, json_payload);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curl_write_callback_response);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response);
res = curl_easy_perform(curl);
if (res != CURLE_OK) {
fprintf(stderr, "curl failed: %s\n", curl_easy_strerror(res));
free(response.data);
curl_slist_free_all(headers);
curl_easy_cleanup(curl);
return NULL;
}
curl_slist_free_all(headers);
curl_easy_cleanup(curl);
}
// Parse AI response from complete response
char *answer = find_ai_response(response.data);
free(response.data);
return answer;
}
BIN
View File
Binary file not shown.
+45
View File
@@ -0,0 +1,45 @@
#include "audio.h"
#include "deepgram_client.h"
#include "intelligence.h"
#include "utils.h"
#include <stdio.h>
#include <stdlib.h>
int main(int argc, char *argv[]) {
char *capture_audio_file = audio_capture(5);
// Reopen file for reading and transcribe
FILE *file = fopen(capture_audio_file, "rb");
if (file != NULL) {
char *transcript = dg_transcribe(file);
if (transcript) {
printf("Transcript: %s\n", transcript);
char *ai_answer = ai_ask(transcript);
if (ai_answer) {
printf("AI answer: %s\n", ai_answer);
struct curl_response_data *tts_response = dg_text_to_speech(ai_answer);
int result = audio_play(tts_response->data, tts_response->size);
if (result != 0) {
fprintf(stderr, "failure in playback");
}
free(tts_response->data);
free(tts_response);
// we have this response data, and buffer within it. we can loop through
// it and write the pcm data within it to alsa
free(ai_answer);
}
free(transcript);
} else {
fprintf(stderr, "unable to get transcript");
}
fclose(file);
} else {
fprintf(stderr, "unable to open file for reading\n");
}
return 0;
}
Binary file not shown.
+30
View File
@@ -0,0 +1,30 @@
#include "utils.h"
#include <string.h>
size_t curl_read_callback_file(void *ptr, size_t size, size_t nmemb,
void *userdata) {
struct curl_file_data *data = (struct curl_file_data *)userdata;
size_t bytes_to_read = size * nmemb;
return fread(ptr, 1, bytes_to_read, data->file);
}
size_t curl_write_callback_response(void *contents, size_t size, size_t nmemb,
void *userp) {
size_t total_size = size * nmemb;
struct curl_response_data *response = (struct curl_response_data *)userp;
// Reallocate buffer to fit new data
response->data = realloc(response->data, response->size + total_size + 1);
if (response->data == NULL) {
return 0; // Error
}
// Copy new data to buffer
memcpy(&response->data[response->size], contents, total_size);
response->size += total_size;
response->data[response->size] = '\0'; // Null terminate
printf("read an extra %d bytes of data\n", (int)total_size);
return total_size;
}