Examples

1
import React, { useState, useEffect } from 'react';
2
import { View, TextInput, Button, Text, FlatList } from 'react-native';
3
import {
4
  isAvailable,
5
  prepareBuiltInModel,
6
  sendMessage,
7
  type LLMMessage,
8
} from 'expo-ai-kit';
9
10
export default function ChatScreen() {
11
  const [messages, setMessages] = useState<LLMMessage[]>([]);
12
  const [input, setInput] = useState('');
13
  const [loading, setLoading] = useState(false);
14
  const [available, setAvailable] = useState(false);
15
16
  useEffect(() => {
17
    void (async () => {
18
      if (!(await isAvailable())) return;
19
      await prepareBuiltInModel();
20
      setAvailable(true);
21
    })();
22
  }, []);
23
24
  const handleSend = async () => {
25
    if (!input.trim() || loading || !available) return;
26
27
    const next: LLMMessage[] = [...messages, { role: 'user', content: input.trim() }];
28
    setMessages(next);
29
    setInput('');
30
    setLoading(true);
31
32
    try {
33
      const response = await sendMessage(next, {
34
        systemPrompt: 'You are a helpful assistant.',
35
      });
36
      setMessages([...next, { role: 'assistant', content: response.text }]);
37
    } catch (error) {
38
      console.error('Error:', error);
39
    } finally {
40
      setLoading(false);
41
    }
42
  };
43
44
  if (!available) {
45
    return (
46
      <View style={{ flex: 1, justifyContent: 'center', alignItems: 'center' }}>
47
        <Text>On-device AI is not available on this device</Text>
48
      </View>
49
    );
50
  }
51
52
  return (
53
    <View style={{ flex: 1, padding: 16 }}>
54
      <FlatList
55
        data={messages}
56
        keyExtractor={(_, i) => i.toString()}
57
        renderItem={({ item }) => (
58
          <View style={{
59
            padding: 12,
60
            marginVertical: 4,
61
            backgroundColor: item.role === 'user' ? '#007AFF' : '#E5E5EA',
62
            borderRadius: 16,
63
            alignSelf: item.role === 'user' ? 'flex-end' : 'flex-start',
64
            maxWidth: '80%',
65
          }}>
66
            <Text style={{ color: item.role === 'user' ? '#fff' : '#000' }}>
67
              {item.content}
68
            </Text>
69
          </View>
70
        )}
71
      />
72
      <View style={{ flexDirection: 'row', gap: 8 }}>
73
        <TextInput
74
          value={input}
75
          onChangeText={setInput}
76
          placeholder="Type a message..."
77
          style={{ flex: 1, borderWidth: 1, borderRadius: 8, padding: 12 }}
78
        />
79
        <Button title={loading ? '...' : 'Send'} onPress={handleSend} />
80
      </View>
81
    </View>
82
  );
83
}

import { generateObject } from 'expo-ai-kit';

type Ticket = {
  sentiment: 'positive' | 'neutral' | 'negative';
  topics: string[];
  urgent: boolean;
};

const { object } = await generateObject<Ticket>(
  [{ role: 'user', content: 'The app keeps crashing on launch and I am furious.' }],
  {
    type: 'object',
    properties: {
      sentiment: { type: 'string', enum: ['positive', 'neutral', 'negative'] },
      topics: { type: 'array', items: { type: 'string' } },
      urgent: { type: 'boolean' },
    },
    required: ['sentiment', 'urgent'],
  },
);

console.log(object.sentiment); // "negative"
console.log(object.urgent);    // true

import { generateText } from 'expo-ai-kit';

const { text, toolCalls } = await generateText(
  [{ role: 'user', content: 'Is it jacket weather in Paris right now?' }],
  {
    tools: {
      getWeather: {
        description: 'Get the current weather for a city.',
        parameters: {
          type: 'object',
          properties: { city: { type: 'string' } },
          required: ['city'],
        },
        execute: async ({ city }: { city: string }) => {
          const res = await fetch(`https://api.example.com/weather?city=${city}`);
          return res.json(); // { tempC, conditions }
        },
      },
    },
    maxSteps: 5,
  },
);

console.log(toolCalls); // [{ toolName: 'getWeather', args: { city: 'Paris' } }]
console.log(text);      // "Yes, it's 11°C and overcast, bring a jacket."

1
import { useState, useRef } from 'react';
2
import { View, Text, Button } from 'react-native';
3
import { streamMessage } from 'expo-ai-kit';
4
5
function ChatWithStreaming() {
6
  const [text, setText] = useState('');
7
  const [isStreaming, setIsStreaming] = useState(false);
8
  const stopRef = useRef<(() => void) | null>(null);
9
10
  const handleSend = async () => {
11
    setIsStreaming(true);
12
    setText('');
13
14
    const { promise, stop } = streamMessage(
15
      [{ role: 'user', content: 'Write a long story' }],
16
      (event) => setText(event.accumulatedText)
17
    );
18
19
    stopRef.current = stop;
20
    await promise;
21
    stopRef.current = null;
22
    setIsStreaming(false);
23
  };
24
25
  const handleStop = () => {
26
    stopRef.current?.();
27
    setIsStreaming(false);
28
  };
29
30
  return (
31
    <View>
32
      <Text>{text}</Text>
33
      {isStreaming ? (
34
        <Button title="Stop" onPress={handleStop} />
35
      ) : (
36
        <Button title="Send" onPress={handleSend} />
37
      )}
38
    </View>
39
  );
40
}

import { useRef, useState } from 'react';
import { Pressable, Text } from 'react-native';
import {
  getSpeechRecognitionAvailability,
  prepareSpeechRecognition,
  requestSpeechPermissionsAsync,
  streamTranscription,
  type TranscriptionHandle,
} from 'expo-ai-kit';

export function PushToTalk() {
  const [text, setText] = useState('');
  const handle = useRef<TranscriptionHandle | null>(null);

  const start = async () => {
    const availability = await getSpeechRecognitionAvailability();
    if (availability.status === 'downloadable') await prepareSpeechRecognition();
    if (!(await requestSpeechPermissionsAsync()).granted) return;
    handle.current = streamTranscription((update) => setText(update.text));
  };

  const stop = () => handle.current?.stop();

  return (
    <Pressable onPressIn={start} onPressOut={stop}>
      <Text>{text || 'Hold to talk'}</Text>
    </Pressable>
  );
}

1
import * as ImagePicker from 'expo-image-picker';
2
import { useState } from 'react';
3
import { Button, Image, Text, View } from 'react-native';
4
import {
5
  getVisionAvailability,
6
  labelImage,
7
  prepareVision,
8
  recognizeText,
9
  removeBackground,
10
  ModelError,
11
} from 'expo-ai-kit';
12
13
export function PhotoInspector() {
14
  const [cutoutUri, setCutoutUri] = useState<string | null>(null);
15
  const [summary, setSummary] = useState('');
16
17
  const inspect = async () => {
18
    const picked = await ImagePicker.launchImageLibraryAsync({ mediaTypes: ['images'] });
19
    if (picked.canceled) return;
20
    const image = { uri: picked.assets[0].uri };
21
22
    // Android downloads its Play services models once; iOS resolves immediately.
23
    const availability = await getVisionAvailability();
24
    if (availability.backgroundRemoval.status === 'downloadable') {
25
      await prepareVision({ features: ['background-removal', 'text-recognition'] });
26
    }
27
28
    const [labels, { text }] = await Promise.all([
29
      labelImage(image, { maxResults: 3 }),
30
      recognizeText(image),
31
    ]);
32
    setSummary(`${labels.map((l) => l.label).join(', ')}
33
${text}`);
34
35
    try {
36
      const cutout = await removeBackground(image);
37
      setCutoutUri(cutout.uri); // PNG with a transparent background, in the app cache
38
    } catch (e) {
39
      if (e instanceof ModelError && e.code === 'NO_SUBJECT_FOUND') {
40
        setCutoutUri(null); // a landscape or document, nothing to cut out
41
      } else {
42
        throw e;
43
      }
44
    }
45
  };
46
47
  return (
48
    <View>
49
      <Button title="Pick a photo" onPress={inspect} />
50
      {cutoutUri && <Image source={{ uri: cutoutUri }} style={{ width: 200, height: 200 }} resizeMode="contain" />}
51
      <Text>{summary}</Text>
52
    </View>
53
  );
54
}

import { generateObject, transcribe } from 'expo-ai-kit';

type Summary = { title: string; actionItems: string[] };

export async function summarizeMemo(uri: string): Promise<Summary> {
  const { text } = await transcribe({ audio: { uri } });
  const { object } = await generateObject<Summary>(
    [{ role: 'user', content: `Summarize this voice memo. Keep action items short.\n\n${text}` }],
    {
      type: 'object',
      properties: {
        title: { type: 'string' },
        actionItems: { type: 'array', items: { type: 'string' } },
      },
      required: ['title', 'actionItems'],
    }
  );
  return object;
}

import { generateObject, recognizeText } from 'expo-ai-kit';

type Receipt = { merchant: string; total: number; date?: string };

export async function scanReceipt(uri: string): Promise<Receipt> {
  const { text } = await recognizeText({ uri });
  const { object } = await generateObject<Receipt>(
    [{ role: 'user', content: `Extract the merchant, total, and date from this receipt:\n\n${text}` }],
    {
      type: 'object',
      properties: {
        merchant: { type: 'string' },
        total: { type: 'number' },
        date: { type: 'string' },
      },
      required: ['merchant', 'total'],
    }
  );
  return object;
}

import {
  getRecommendedModel,
  downloadModel,
  setModel,
  type DownloadableModel,
} from 'expo-ai-kit';

export async function setupBestModel(onProgress: (p: number) => void) {
  const best: DownloadableModel | null = await getRecommendedModel();
  if (!best) return null; // device can't run any downloadable model

  if (best.status !== 'downloaded' && best.status !== 'ready') {
    await downloadModel(best.id, { onProgress });
  }

  await setModel(best.id, { generation: { temperature: 0.7 } });
  return best.id; // sendMessage / generateObject / generateText now use it
}

import {
  isAvailable,
  prepareBuiltInModel,
  sendMessage,
  ModelError,
  type LLMMessage,
} from 'expo-ai-kit';

export async function safeAIRequest(messages: LLMMessage[], systemPrompt?: string) {
  if (!(await isAvailable())) {
    return { success: false as const, error: 'On-device AI is not available' };
  }

  try {
    await prepareBuiltInModel();
    const { text } = await sendMessage(messages, { systemPrompt });
    return { success: true as const, result: text };
  } catch (e) {
    if (e instanceof ModelError) {
      // e.code: 'INFERENCE_BUSY' | 'INFERENCE_OOM' | 'MODEL_NOT_DOWNLOADED' | ...
      return { success: false as const, error: `${e.code}: ${e.message}` };
    }
    return { success: false as const, error: 'Unknown error' };
  }
}