Integrating TOON in Your AI Application

export interface ToonData {
  schema: string[];
  rows: string[][];
}

export function jsonToToon(data: any[]): string {
  if (!Array.isArray(data) || data.length === 0) {
    throw new Error('Input must be non-empty array');
  }

  // Extract schema
  const schema = Object.keys(data[0]);
  
  // Build TOON
  let toon = `TOON.schema: ${schema.join(' | ')}\n`;
  
  for (const obj of data) {
    const values = schema.map(key => String(obj[key] ?? ''));
    toon += values.join(' | ') + '\n';
  }
  
  return toon;
}

export function parseToon(toonStr: string): ToonData {
  const lines = toonStr.trim().split('\n');
  
  // Parse schema
  const schemaLine = lines[0];
  if (!schemaLine.startsWith('TOON.schema:')) {
    throw new Error('Invalid TOON format');
  }
  
  const schema = schemaLine
    .replace('TOON.schema:', '')
    .split('|')
    .map(s => s.trim());
  
  // Parse rows
  const rows = lines.slice(1).map(line =>
    line.split('|').map(s => s.trim())
  );
  
  return { schema, rows };
}

// Convert back to JSON
export function toonToJson(toonStr: string): any[] {
  const { schema, rows } = parseToon(toonStr);
  
  return rows.map(row => {
    const obj: any = {};
    schema.forEach((key, i) => {
      obj[key] = row[i];
    });
    return obj;
  });
}

from langchain.schema import Document
from langchain.vectorstores import Chroma
from langchain.embeddings import OpenAIEmbeddings
from langchain.chat_models import ChatOpenAI
from langchain.chains import RetrievalQA

def create_toon_documents(data, chunk_size=50):
    """Split large dataset into TOON chunks"""
    documents = []
    
    for i in range(0, len(data), chunk_size):
        chunk = data[i:i + chunk_size]
        toon_text = json_to_toon(chunk)
        
        doc = Document(
            page_content=toon_text,
            metadata={"format": "toon", "rows": len(chunk)}
        )
        documents.append(doc)
    
    return documents

# Example: RAG with TOON
products = fetch_products_from_db()  # 10,000 products

# Convert to TOON documents
toon_docs = create_toon_documents(products, chunk_size=100)

# Create vector store
embeddings = OpenAIEmbeddings()
vectorstore = Chroma.from_documents(toon_docs, embeddings)

# Create QA chain
llm = ChatOpenAI(model="gpt-4", temperature=0)
qa_chain = RetrievalQA.from_chain_type(
    llm=llm,
    chain_type="stuff",
    retriever=vectorstore.as_retriever(search_kwargs={"k": 3})
)

# Query
result = qa_chain.run("What are the top-rated laptops under $1000?")
print(result)