How to Integrate Bolt.new with TensorFlow

1// hooks/useMobileNet.ts
2import { useState, useEffect, useCallback, useRef } from 'react';
3import * as tf from '@tensorflow/tfjs';
4import * as mobilenet from '@tensorflow-models/mobilenet';
5
6interface Prediction {
7  className: string;
8  probability: number;
9}
10
11export function useMobileNet() {
12  const modelRef = useRef<mobilenet.MobileNet | null>(null);
13  const [loading, setLoading] = useState(true);
14  const [error, setError] = useState<string | null>(null);
15
16  useEffect(() => {
17    let cancelled = false;
18
19    async function loadModel() {
20      try {
21        // Prefer WebGL backend for GPU acceleration
22        await tf.setBackend('webgl');
23        const model = await mobilenet.load({ version: 2, alpha: 1.0 });
24        if (!cancelled) {
25          modelRef.current = model;
26          setLoading(false);
27        }
28      } catch (err) {
29        if (!cancelled) {
30          setError(err instanceof Error ? err.message : 'Failed to load model');
31          setLoading(false);
32        }
33      }
34    }
35
36    loadModel();
37    return () => { cancelled = true; };
38  }, []);
39
40  const classify = useCallback(
41    async (imageElement: HTMLImageElement | HTMLCanvasElement | HTMLVideoElement): Promise<Prediction[]> => {
42      if (!modelRef.current) throw new Error('Model not loaded');
43      const predictions = await modelRef.current.classify(imageElement, 5);
44      return predictions;
45    },
46    []
47  );
48
49  return { loading, error, classify };
50}

1// components/ImageClassifier.tsx
2'use client';
3import { useState, useRef, useCallback } from 'react';
4import { useMobileNet } from '@/hooks/useMobileNet';
5
6interface Prediction { className: string; probability: number; }
7
8export function ImageClassifier() {
9  const { loading: modelLoading, error: modelError, classify } = useMobileNet();
10  const [imageUrl, setImageUrl] = useState<string | null>(null);
11  const [predictions, setPredictions] = useState<Prediction[]>([]);
12  const [classifying, setClassifying] = useState(false);
13  const imgRef = useRef<HTMLImageElement>(null);
14  const prevUrlRef = useRef<string | null>(null);
15
16  const handleFileChange = useCallback((file: File) => {
17    if (prevUrlRef.current) URL.revokeObjectURL(prevUrlRef.current);
18    const url = URL.createObjectURL(file);
19    prevUrlRef.current = url;
20    setImageUrl(url);
21    setPredictions([]);
22  }, []);
23
24  const handleClassify = useCallback(async () => {
25    if (!imgRef.current || !imageUrl) return;
26    setClassifying(true);
27    try {
28      const results = await classify(imgRef.current);
29      setPredictions(results);
30    } finally {
31      setClassifying(false);
32    }
33  }, [classify, imageUrl]);
34
35  const confidenceColor = (p: number) =>
36    p >= 0.6 ? 'bg-green-500' : p >= 0.3 ? 'bg-yellow-500' : 'bg-red-400';
37
38  if (modelLoading) return <div className="p-6 text-center text-gray-500">Loading MobileNet model...</div>;
39  if (modelError) return <div className="p-6 text-red-600">Model error: {modelError}</div>;
40
41  return (
42    <div className="max-w-lg mx-auto p-6 space-y-4">
43      <h2 className="text-2xl font-bold">Image Classifier</h2>
44      <p className="text-sm text-gray-500">Powered by TensorFlow.js MobileNet — runs entirely in your browser</p>
45
46      <label className="block border-2 border-dashed rounded-lg p-6 text-center cursor-pointer hover:bg-gray-50">
47        <span className="text-gray-600">Click to upload or drag an image here</span>
48        <input type="file" accept="image/*" className="hidden"
49          onChange={(e) => e.target.files?.[0] && handleFileChange(e.target.files[0])} />
50      </label>
51
52      {imageUrl && (
53        <>
54          <img ref={imgRef} src={imageUrl} alt="To classify"
55            className="w-full rounded-lg max-h-64 object-contain" />
56          <button onClick={handleClassify} disabled={classifying}
57            className="w-full bg-blue-600 text-white py-2 rounded hover:bg-blue-700 disabled:opacity-50">
58            {classifying ? 'Analyzing...' : 'Classify Image'}
59          </button>
60        </>
61      )}
62
63      {predictions.length > 0 && (
64        <div className="space-y-2">
65          <h3 className="font-semibold">Results</h3>
66          {predictions.map((p) => (
67            <div key={p.className}>
68              <div className="flex justify-between text-sm mb-1">
69                <span>{p.className}</span>
70                <span>{(p.probability * 100).toFixed(1)}%</span>
71              </div>
72              <div className="w-full bg-gray-200 rounded-full h-2">
73                <div className={`h-2 rounded-full ${confidenceColor(p.probability)}`}
74                  style={{ width: `${p.probability * 100}%` }} />
75              </div>
76            </div>
77          ))}
78        </div>
79      )}
80    </div>
81  );
82}

1// components/ObjectDetector.tsx
2'use client';
3import { useState, useRef, useEffect, useCallback } from 'react';
4import * as tf from '@tensorflow/tfjs';
5import * as cocoSsd from '@tensorflow-models/coco-ssd';
6
7const COLORS = ['#EF4444','#3B82F6','#10B981','#F59E0B','#8B5CF6','#EC4899'];
8
9export function ObjectDetector() {
10  const videoRef = useRef<HTMLVideoElement>(null);
11  const canvasRef = useRef<HTMLCanvasElement>(null);
12  const modelRef = useRef<cocoSsd.ObjectDetection | null>(null);
13  const rafRef = useRef<number>(0);
14  const streamRef = useRef<MediaStream | null>(null);
15
16  const [status, setStatus] = useState<'idle' | 'loading' | 'running' | 'error'>('idle');
17  const [detections, setDetections] = useState<cocoSsd.DetectedObject[]>([]);
18  const [errorMsg, setErrorMsg] = useState('');
19
20  const drawDetections = useCallback((preds: cocoSsd.DetectedObject[]) => {
21    const canvas = canvasRef.current;
22    const video = videoRef.current;
23    if (!canvas || !video) return;
24    const ctx = canvas.getContext('2d')!;
25    canvas.width = video.videoWidth;
26    canvas.height = video.videoHeight;
27    ctx.clearRect(0, 0, canvas.width, canvas.height);
28
29    preds.forEach((pred, i) => {
30      const [x, y, w, h] = pred.bbox;
31      const color = COLORS[i % COLORS.length];
32      ctx.strokeStyle = color;
33      ctx.lineWidth = 3;
34      ctx.strokeRect(x, y, w, h);
35      ctx.fillStyle = color;
36      ctx.fillRect(x, y - 22, w, 22);
37      ctx.fillStyle = '#fff';
38      ctx.font = '14px sans-serif';
39      ctx.fillText(`${pred.class} ${(pred.score * 100).toFixed(0)}%`, x + 4, y - 6);
40    });
41  }, []);
42
43  const detect = useCallback(async () => {
44    if (!modelRef.current || !videoRef.current || videoRef.current.readyState < 2) {
45      rafRef.current = requestAnimationFrame(detect);
46      return;
47    }
48    const preds = await modelRef.current.detect(videoRef.current);
49    drawDetections(preds);
50    setDetections(preds);
51    rafRef.current = requestAnimationFrame(detect);
52  }, [drawDetections]);
53
54  const startCamera = useCallback(async () => {
55    setStatus('loading');
56    try {
57      await tf.setBackend('webgl');
58      modelRef.current = await cocoSsd.load();
59      const stream = await navigator.mediaDevices.getUserMedia({ video: true });
60      streamRef.current = stream;
61      videoRef.current!.srcObject = stream;
62      await videoRef.current!.play();
63      setStatus('running');
64      rafRef.current = requestAnimationFrame(detect);
65    } catch (err) {
66      setStatus('error');
67      setErrorMsg(err instanceof Error ? err.message : 'Camera or model error');
68    }
69  }, [detect]);
70
71  const stopCamera = useCallback(() => {
72    cancelAnimationFrame(rafRef.current);
73    streamRef.current?.getTracks().forEach((t) => t.stop());
74    streamRef.current = null;
75    setStatus('idle');
76    setDetections([]);
77  }, []);
78
79  useEffect(() => () => stopCamera(), [stopCamera]);
80
81  return (
82    <div className="max-w-2xl mx-auto p-6 space-y-4">
83      <h2 className="text-2xl font-bold">Real-Time Object Detection</h2>
84      <p className="text-sm text-gray-500">TensorFlow.js COCO-SSD — runs in your browser, no API key needed</p>
85
86      <div className="relative bg-black rounded-lg overflow-hidden" style={{ aspectRatio: '16/9' }}>
87        <video ref={videoRef} className="w-full h-full object-cover" muted playsInline />
88        <canvas ref={canvasRef} className="absolute inset-0 w-full h-full" />
89        {status === 'idle' && (
90          <div className="absolute inset-0 flex items-center justify-center">
91            <button onClick={startCamera} className="bg-blue-600 text-white px-6 py-3 rounded-lg text-lg">
92              Start Camera
93            </button>
94          </div>
95        )}
96        {status === 'loading' && (
97          <div className="absolute inset-0 flex items-center justify-center bg-black/50">
98            <p className="text-white">Loading COCO-SSD model...</p>
99          </div>
100        )}
101      </div>
102
103      {status === 'error' && <p className="text-red-600">Error: {errorMsg}</p>}
104
105      {status === 'running' && (
106        <button onClick={stopCamera} className="w-full border border-red-300 text-red-600 py-2 rounded hover:bg-red-50">
107          Stop Camera
108        </button>
109      )}
110
111      {detections.length > 0 && (
112        <div>
113          <h3 className="font-semibold mb-2">Detected ({detections.length})</h3>
114          <div className="flex flex-wrap gap-2">
115            {detections.map((d, i) => (
116              <span key={i} className="px-3 py-1 bg-blue-100 text-blue-800 rounded-full text-sm">
117                {d.class} {(d.score * 100).toFixed(0)}%
118              </span>
119            ))}
120          </div>
121        </div>
122      )}
123    </div>
124  );
125}

1// components/CustomModelRunner.tsx
2'use client';
3import { useState, useRef } from 'react';
4import * as tf from '@tensorflow/tfjs';
5
6export function CustomModelRunner() {
7  const modelRef = useRef<tf.LayersModel | null>(null);
8  const [modelUrl, setModelUrl] = useState('');
9  const [featureInput, setFeatureInput] = useState('');
10  const [modelInfo, setModelInfo] = useState<string | null>(null);
11  const [prediction, setPrediction] = useState<number[] | null>(null);
12  const [memStats, setMemStats] = useState<tf.MemoryInfo | null>(null);
13  const [loading, setLoading] = useState(false);
14
15  const loadModel = async () => {
16    if (!modelUrl) return;
17    setLoading(true);
18    try {
19      await tf.setBackend('webgl');
20      const model = await tf.loadLayersModel(modelUrl);
21      modelRef.current = model;
22      const inputShape = model.inputs[0].shape.join(' x ');
23      const outputShape = model.outputs[0].shape.join(' x ');
24      setModelInfo(`Input: [${inputShape}] | Output: [${outputShape}]`);
25    } catch (err) {
26      setModelInfo(`Load error: ${err instanceof Error ? err.message : 'Failed'}`);
27    } finally {
28      setLoading(false);
29    }
30  };
31
32  const runPrediction = () => {
33    if (!modelRef.current || !featureInput) return;
34    try {
35      const features = featureInput.split(',').map(Number);
36      const inputShape = modelRef.current.inputs[0].shape.slice(1) as number[];
37      const result = tf.tidy(() => {
38        const tensor = tf.tensor2d([features], [1, ...inputShape]);
39        const output = modelRef.current!.predict(tensor) as tf.Tensor;
40        return Array.from(output.dataSync());
41      });
42      setPrediction(result);
43      setMemStats(tf.memory());
44    } catch (err) {
45      setPrediction(null);
46      setModelInfo(`Prediction error: ${err instanceof Error ? err.message : 'Failed'}`);
47    }
48  };
49
50  return (
51    <div className="max-w-lg mx-auto p-6 space-y-4">
52      <h2 className="text-2xl font-bold">Custom TF.js Model</h2>
53      <div className="flex gap-2">
54        <input value={modelUrl} onChange={(e) => setModelUrl(e.target.value)}
55          placeholder="https://example.com/model.json" className="flex-1 border rounded px-3 py-2" />
56        <button onClick={loadModel} disabled={loading}
57          className="bg-blue-600 text-white px-4 py-2 rounded disabled:opacity-50">
58          {loading ? 'Loading...' : 'Load'}
59        </button>
60      </div>
61      {modelInfo && <p className="text-sm font-mono bg-gray-100 p-2 rounded">{modelInfo}</p>}
62      <input value={featureInput} onChange={(e) => setFeatureInput(e.target.value)}
63        placeholder="Feature values, comma-separated (e.g. 0.5, 1.2, 3.0)" className="w-full border rounded px-3 py-2" />
64      <button onClick={runPrediction} disabled={!modelRef.current}
65        className="w-full bg-green-600 text-white py-2 rounded disabled:opacity-50">
66        Run Prediction
67      </button>
68      {prediction && (
69        <div className="bg-gray-50 rounded p-3">
70          <p className="font-semibold text-sm">Output:</p>
71          <pre className="text-xs mt-1">{JSON.stringify(prediction, null, 2)}</pre>
72          {memStats && <p className="text-xs text-gray-500 mt-2">Tensors: {memStats.numTensors} | GPU bytes: {(memStats.numBytesInGPU ?? 0).toLocaleString()}</p>}
73        </div>
74      )}
75    </div>
76  );
77}