Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| db053c2dbd |
@@ -271,9 +271,13 @@ Die Bridge verbindet die Android App mit ARIA und bietet lokale Sprachverarbeitu
|
||||
|
||||
**Nachrichtenfluss:**
|
||||
```
|
||||
App → RVS → Bridge → aria-core
|
||||
aria-core → Bridge → RVS → App
|
||||
→ Lautsprecher (TTS)
|
||||
Text: App → RVS → Bridge → chat.send → aria-core
|
||||
Audio: App → RVS → Bridge → FFmpeg → Whisper STT → chat.send → aria-core
|
||||
Datei: App → RVS → Bridge → /shared/uploads/ → chat.send (mit Pfad) → aria-core
|
||||
|
||||
aria-core → Antwort → Gateway → Diagnostic → RVS → App
|
||||
→ Bridge → Piper TTS → RVS → App (Audio)
|
||||
→ Bridge → Lautsprecher (lokal)
|
||||
```
|
||||
|
||||
### Features
|
||||
@@ -335,9 +339,11 @@ API-Endpoint fuer andere Services: `GET http://localhost:3001/api/session`
|
||||
- Text-Chat mit ARIA
|
||||
- **Sprachaufnahme**: Push-to-Talk (halten) oder Tap-to-Talk (tippen, Auto-Stop bei Stille)
|
||||
- **VAD (Voice Activity Detection)**: Erkennt 1.8s Stille und stoppt automatisch
|
||||
- **Wake Word**: Toggle-Button aktiviert kontinuierliches Mikrofon-Monitoring
|
||||
- **STT (Speech-to-Text)**: Audio wird in der Bridge per Whisper transkribiert, transkribierter Text erscheint im Chat
|
||||
- **Wake Word**: Toggle-Button (Ohr-Symbol) aktiviert kontinuierliches Mikrofon-Monitoring
|
||||
- **TTS-Wiedergabe**: ARIA antwortet per Lautsprecher (Ramona/Thorsten)
|
||||
- Datei- und Kamera-Upload
|
||||
- **Datei- und Bild-Upload**: Bilder inline im Chat, Dateien mit Icon + Name + Groesse
|
||||
- **Anhaenge**: Bridge speichert Dateien in Shared Volume (`/shared/uploads/`), ARIA kann darauf zugreifen
|
||||
- GPS-Position (optional)
|
||||
- QR-Code Scanner fuer Token-Pairing
|
||||
|
||||
@@ -381,15 +387,28 @@ GITEA_REPO=stefan/aria-agent
|
||||
GITEA_USER=stefan
|
||||
```
|
||||
|
||||
### Audio-Pipeline
|
||||
### Audio-Pipeline (Spracheingabe)
|
||||
|
||||
```
|
||||
App (Mikrofon) → AAC/MP4 Aufnahme → Base64 → RVS → Bridge
|
||||
Bridge: FFmpeg (16kHz PCM) → Whisper STT → Text → aria-core
|
||||
Bridge: STT-Ergebnis → RVS → App (Placeholder wird durch transkribierten Text ersetzt)
|
||||
aria-core → Antwort → Bridge → Piper TTS (WAV) → Base64 → RVS → App
|
||||
App: Base64 → WAV → Lautsprecher
|
||||
```
|
||||
|
||||
### Datei-Pipeline (Bilder & Anhaenge)
|
||||
|
||||
```
|
||||
App (Kamera/Dateimanager) → Base64 → RVS → Bridge
|
||||
Bridge: Speichert in /shared/uploads/ (Shared Volume, fuer aria-core sichtbar)
|
||||
Bridge: chat.send → "Stefan hat ein Bild geschickt: foto.jpg — liegt unter /shared/uploads/..."
|
||||
ARIA: Kann Datei per Bash/Read-Tool oeffnen und analysieren
|
||||
```
|
||||
|
||||
**Unterstuetzte Formate:** Bilder (JPG, PNG), Dokumente (PDF, DOCX, TXT), beliebige Dateien.
|
||||
Bilder werden in der App inline angezeigt, andere Dateien als Icon + Dateiname.
|
||||
|
||||
---
|
||||
|
||||
## Datenverzeichnis — aria-data/
|
||||
@@ -453,6 +472,8 @@ docker compose up -d
|
||||
| `./aria-data/ssh` (bind) | `/root/.ssh`, `/home/node/.ssh` | SSH Keys |
|
||||
| `./aria-data/brain` (bind) | `/home/node/.openclaw/workspace/memory` | Gedaechtnis |
|
||||
| `./aria-data/skills` (bind) | `/home/node/.openclaw/workspace/skills` | Skills |
|
||||
| `aria-shared` | `/shared` (Core + Bridge) | Datei-Austausch (Uploads von App) |
|
||||
| `./aria-data/config/diag-state` (bind) | `/data` (Diagnostic) | Persistenter State (aktive Session) |
|
||||
|
||||
---
|
||||
|
||||
@@ -507,8 +528,13 @@ docker exec aria-core ssh aria-wohnung hostname
|
||||
Dadurch ist ARIA langsamer als die direkte Claude CLI. Timeout ist auf 900s (15 Min).
|
||||
- **Kein Streaming zur App**: Die App zeigt erst die fertige Antwort, keine Streaming-Tokens.
|
||||
- **Wake Word nur auf VM**: Die Bridge hoert auf "ARIA" ueber das lokale Mikrofon der VM.
|
||||
In der App gibt es Energy-basierte Erkennung (Phase 1).
|
||||
In der App gibt es Energy-basierte Erkennung (Phase 1). On-device "ARIA"-Keyword (Porcupine) ist Phase 2.
|
||||
- **Audio-Format**: App nimmt AAC/MP4 auf, Bridge konvertiert via FFmpeg zu 16kHz PCM.
|
||||
- **Bildanalyse eingeschraenkt**: Bilder werden in `/shared/uploads/` gespeichert. ARIA kann
|
||||
sie per Bash/Read-Tool oeffnen, aber Claude Vision (direkte Bildanalyse) ist ueber den
|
||||
Proxy-Pfad (`claude --print`) noch nicht moeglich. ARIA sieht den Dateipfad, nicht das Bild.
|
||||
- **Dateigroesse**: Grosse Dateien (>5MB) koennen WebSocket-Limits ueberschreiten.
|
||||
Bilder werden in der App auf max 1920x1920px @ 80% Qualitaet komprimiert.
|
||||
|
||||
---
|
||||
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
BIN
Binary file not shown.
+1
-1
File diff suppressed because one or more lines are too long
Binary file not shown.
BIN
Binary file not shown.
+1
-1
@@ -1,4 +1,4 @@
|
||||
#Sun Mar 29 12:08:20 CEST 2026
|
||||
#Sun Mar 29 12:31:28 CEST 2026
|
||||
base.2=/home/duffy/Dokumente/programmierung/ARIA-AGENT/android/android/app/build/intermediates/dex/release/mergeDexRelease/classes2.dex
|
||||
path.2=classes2.dex
|
||||
base.1=/home/duffy/Dokumente/programmierung/ARIA-AGENT/android/android/app/build/intermediates/global_synthetics_dex/release/classes.dex
|
||||
|
||||
+1
-1
File diff suppressed because one or more lines are too long
+1
-1
File diff suppressed because one or more lines are too long
@@ -15,6 +15,7 @@ import {
|
||||
KeyboardAvoidingView,
|
||||
Platform,
|
||||
StyleSheet,
|
||||
Image,
|
||||
Modal,
|
||||
} from 'react-native';
|
||||
import AsyncStorage from '@react-native-async-storage/async-storage';
|
||||
@@ -33,6 +34,8 @@ interface Attachment {
|
||||
type: 'image' | 'file' | 'audio';
|
||||
name: string;
|
||||
size?: number;
|
||||
uri?: string; // Lokaler Pfad oder data URI fuer Anzeige
|
||||
mimeType?: string;
|
||||
}
|
||||
|
||||
interface ChatMessage {
|
||||
@@ -87,10 +90,7 @@ const ChatScreen: React.FC = () => {
|
||||
console.error('[Chat] Fehler beim Laden des Verlaufs:', err);
|
||||
}
|
||||
};
|
||||
loadMessages().then(() => {
|
||||
// Auto-Scroll nach Laden des Verlaufs
|
||||
setTimeout(() => flatListRef.current?.scrollToEnd({ animated: false }), 200);
|
||||
});
|
||||
loadMessages();
|
||||
}, []);
|
||||
|
||||
// RVS-Nachrichten abonnieren
|
||||
@@ -222,19 +222,22 @@ const ChatScreen: React.FC = () => {
|
||||
);
|
||||
}, [messages]);
|
||||
|
||||
// Auto-Scroll bei neuen Nachrichten
|
||||
useEffect(() => {
|
||||
if (messages.length > 0) {
|
||||
// Laengerer Delay damit FlatList fertig gerendert hat
|
||||
setTimeout(() => {
|
||||
flatListRef.current?.scrollToEnd({ animated: false });
|
||||
}, 300);
|
||||
// Nochmal animiert fuer den Fall dass sich die Hoehe geaendert hat
|
||||
setTimeout(() => {
|
||||
flatListRef.current?.scrollToEnd({ animated: true });
|
||||
}, 600);
|
||||
// Auto-Scroll wird ueber onContentSizeChange der FlatList gesteuert
|
||||
const shouldAutoScroll = useRef(true);
|
||||
const handleContentSizeChange = useCallback(() => {
|
||||
if (shouldAutoScroll.current) {
|
||||
flatListRef.current?.scrollToEnd({ animated: false });
|
||||
}
|
||||
}, [messages]);
|
||||
}, []);
|
||||
const handleScrollBeginDrag = useCallback(() => {
|
||||
shouldAutoScroll.current = false;
|
||||
}, []);
|
||||
const handleScrollEndDrag = useCallback((e: any) => {
|
||||
// Auto-Scroll wieder aktivieren wenn User ganz unten ist
|
||||
const { contentOffset, contentSize, layoutMeasurement } = e.nativeEvent;
|
||||
const isAtBottom = contentOffset.y + layoutMeasurement.height >= contentSize.height - 50;
|
||||
shouldAutoScroll.current = isAtBottom;
|
||||
}, []);
|
||||
|
||||
// GPS-Position holen (optional)
|
||||
const getCurrentLocation = useCallback((): Promise<{ lat: number; lon: number } | null> => {
|
||||
@@ -306,12 +309,19 @@ const ChatScreen: React.FC = () => {
|
||||
setShowFileUpload(false);
|
||||
const location = await getCurrentLocation();
|
||||
|
||||
const isImage = file.type.startsWith('image/');
|
||||
const userMsg: ChatMessage = {
|
||||
id: nextId(),
|
||||
sender: 'user',
|
||||
text: `[Datei: ${file.name}]`,
|
||||
text: 'Anhang empfangen',
|
||||
timestamp: Date.now(),
|
||||
attachments: [{ type: 'file', name: file.name, size: file.size }],
|
||||
attachments: [{
|
||||
type: isImage ? 'image' : 'file',
|
||||
name: file.name,
|
||||
size: file.size,
|
||||
uri: isImage && file.base64 ? `data:${file.type};base64,${file.base64}` : file.uri,
|
||||
mimeType: file.type,
|
||||
}],
|
||||
};
|
||||
setMessages(prev => [...prev, userMsg]);
|
||||
|
||||
@@ -332,9 +342,14 @@ const ChatScreen: React.FC = () => {
|
||||
const userMsg: ChatMessage = {
|
||||
id: nextId(),
|
||||
sender: 'user',
|
||||
text: `[Foto: ${photo.fileName}]`,
|
||||
text: 'Anhang empfangen',
|
||||
timestamp: Date.now(),
|
||||
attachments: [{ type: 'image', name: photo.fileName }],
|
||||
attachments: [{
|
||||
type: 'image',
|
||||
name: photo.fileName,
|
||||
uri: photo.base64 ? `data:${photo.type};base64,${photo.base64}` : undefined,
|
||||
mimeType: photo.type,
|
||||
}],
|
||||
};
|
||||
setMessages(prev => [...prev, userMsg]);
|
||||
|
||||
@@ -359,16 +374,35 @@ const ChatScreen: React.FC = () => {
|
||||
|
||||
return (
|
||||
<View style={[styles.messageBubble, isUser ? styles.userBubble : styles.ariaBubble]}>
|
||||
<Text style={[styles.messageText, isUser ? styles.userText : styles.ariaText]}>
|
||||
{item.text}
|
||||
</Text>
|
||||
{/* Anhang-Vorschau */}
|
||||
{item.attachments?.map((att, idx) => (
|
||||
<View key={idx} style={styles.attachmentBadge}>
|
||||
<Text style={styles.attachmentText}>
|
||||
{att.type === 'image' ? '\uD83D\uDDBC\uFE0F' : att.type === 'audio' ? '\uD83C\uDFA4' : '\uD83D\uDCC4'} {att.name}
|
||||
</Text>
|
||||
<View key={idx}>
|
||||
{att.type === 'image' && att.uri ? (
|
||||
<Image
|
||||
source={{ uri: att.uri }}
|
||||
style={styles.attachmentImage}
|
||||
resizeMode="contain"
|
||||
/>
|
||||
) : (
|
||||
<View style={styles.attachmentFile}>
|
||||
<Text style={styles.attachmentFileIcon}>
|
||||
{att.mimeType?.includes('pdf') ? '\uD83D\uDCC4' :
|
||||
att.mimeType?.includes('word') || att.mimeType?.includes('document') ? '\uD83D\uDCC3' :
|
||||
att.mimeType?.includes('sheet') || att.mimeType?.includes('excel') ? '\uD83D\uDCC8' :
|
||||
'\uD83D\uDCC1'}
|
||||
</Text>
|
||||
<Text style={styles.attachmentFileName} numberOfLines={1}>{att.name}</Text>
|
||||
{att.size ? <Text style={styles.attachmentFileSize}>{Math.round(att.size / 1024)}KB</Text> : null}
|
||||
</View>
|
||||
)}
|
||||
</View>
|
||||
))}
|
||||
{/* Text (nicht anzeigen wenn nur "Anhang empfangen" und ein Bild da ist) */}
|
||||
{!(item.text === 'Anhang empfangen' && item.attachments?.some(a => a.type === 'image' && a.uri)) && (
|
||||
<Text style={[styles.messageText, isUser ? styles.userText : styles.ariaText]}>
|
||||
{item.text}
|
||||
</Text>
|
||||
)}
|
||||
<Text style={styles.timestamp}>{time}</Text>
|
||||
</View>
|
||||
);
|
||||
@@ -401,6 +435,9 @@ const ChatScreen: React.FC = () => {
|
||||
renderItem={renderMessage}
|
||||
contentContainerStyle={styles.messageList}
|
||||
showsVerticalScrollIndicator={false}
|
||||
onContentSizeChange={handleContentSizeChange}
|
||||
onScrollBeginDrag={handleScrollBeginDrag}
|
||||
onScrollEndDrag={handleScrollEndDrag}
|
||||
ListEmptyComponent={
|
||||
<View style={styles.emptyContainer}>
|
||||
<Text style={styles.emptyIcon}>{'\uD83E\uDD16'}</Text>
|
||||
@@ -542,17 +579,34 @@ const styles = StyleSheet.create({
|
||||
ariaText: {
|
||||
color: '#E0E0F0',
|
||||
},
|
||||
attachmentBadge: {
|
||||
backgroundColor: 'rgba(255,255,255,0.1)',
|
||||
borderRadius: 6,
|
||||
paddingHorizontal: 8,
|
||||
paddingVertical: 4,
|
||||
marginTop: 6,
|
||||
alignSelf: 'flex-start',
|
||||
attachmentImage: {
|
||||
width: '100%',
|
||||
height: 200,
|
||||
borderRadius: 8,
|
||||
marginBottom: 6,
|
||||
backgroundColor: '#0D0D1A',
|
||||
},
|
||||
attachmentText: {
|
||||
color: '#CCCCDD',
|
||||
fontSize: 12,
|
||||
attachmentFile: {
|
||||
flexDirection: 'row',
|
||||
alignItems: 'center',
|
||||
backgroundColor: 'rgba(255,255,255,0.1)',
|
||||
borderRadius: 8,
|
||||
padding: 10,
|
||||
marginBottom: 6,
|
||||
},
|
||||
attachmentFileIcon: {
|
||||
fontSize: 24,
|
||||
marginRight: 8,
|
||||
},
|
||||
attachmentFileName: {
|
||||
flex: 1,
|
||||
color: '#E0E0F0',
|
||||
fontSize: 13,
|
||||
},
|
||||
attachmentFileSize: {
|
||||
color: '#8888AA',
|
||||
fontSize: 11,
|
||||
marginLeft: 8,
|
||||
},
|
||||
timestamp: {
|
||||
color: 'rgba(255,255,255,0.4)',
|
||||
|
||||
+50
-34
@@ -964,29 +964,38 @@ class ARIABridge:
|
||||
logger.info("[rvs] Datei empfangen: %s (%s, %dKB)",
|
||||
file_name, file_type, len(file_b64) // 1365 if file_b64 else 0)
|
||||
|
||||
# Shared Volume: /shared/ ist in Bridge UND aria-core gemountet
|
||||
SHARED_DIR = "/shared/uploads"
|
||||
os.makedirs(SHARED_DIR, exist_ok=True)
|
||||
|
||||
if file_b64 and file_type.startswith("image/"):
|
||||
# Bild: als temporaere Datei speichern und Pfad an ARIA melden
|
||||
# Bild in Shared Volume speichern
|
||||
ext = ".jpg" if "jpeg" in file_type or "jpg" in file_type else ".png"
|
||||
tmp = tempfile.NamedTemporaryFile(suffix=ext, dir="/tmp", delete=False, prefix="aria_img_")
|
||||
tmp.write(base64.b64decode(file_b64))
|
||||
tmp.close()
|
||||
text = (f"[Bild von Stefan via App: {file_name}"
|
||||
f"{f', {width}x{height}px' if width else ''}"
|
||||
f" — gespeichert als {tmp.name}]"
|
||||
f" Bitte analysiere das Bild.")
|
||||
safe_name = f"img_{int(asyncio.get_event_loop().time())}_{file_name.replace('/', '_')}"
|
||||
file_path = os.path.join(SHARED_DIR, safe_name if safe_name.endswith(ext) else safe_name + ext)
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(base64.b64decode(file_b64))
|
||||
size_kb = len(file_b64) // 1365
|
||||
logger.info("[rvs] Bild gespeichert: %s (%dKB)", file_path, size_kb)
|
||||
text = (f"Stefan hat dir ein Bild geschickt: {file_name}"
|
||||
f"{f' ({width}x{height}px)' if width else ''}"
|
||||
f", {size_kb}KB."
|
||||
f" Das Bild liegt unter: {file_path}")
|
||||
await self.send_to_core(text, source="app-file")
|
||||
elif file_b64:
|
||||
# Andere Datei: speichern und Pfad melden
|
||||
ext = Path(file_name).suffix or ".bin"
|
||||
tmp = tempfile.NamedTemporaryFile(suffix=ext, dir="/tmp", delete=False, prefix="aria_file_")
|
||||
tmp.write(base64.b64decode(file_b64))
|
||||
tmp.close()
|
||||
text = (f"[Datei von Stefan via App: {file_name}"
|
||||
f" ({file_type}, {file_size} bytes)"
|
||||
f" — gespeichert als {tmp.name}]")
|
||||
# Andere Datei in Shared Volume speichern
|
||||
safe_name = f"file_{int(asyncio.get_event_loop().time())}_{file_name.replace('/', '_')}"
|
||||
file_path = os.path.join(SHARED_DIR, safe_name)
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(base64.b64decode(file_b64))
|
||||
size_kb = len(file_b64) // 1365
|
||||
logger.info("[rvs] Datei gespeichert: %s (%dKB)", file_path, size_kb)
|
||||
text = (f"Stefan hat dir eine Datei geschickt: {file_name}"
|
||||
f" ({file_type}, {size_kb}KB)."
|
||||
f" Die Datei liegt unter: {file_path}")
|
||||
await self.send_to_core(text, source="app-file")
|
||||
else:
|
||||
text = f"[Stefan hat eine Datei gesendet: {file_name} ({file_type}) — aber keine Daten empfangen]"
|
||||
text = f"Stefan hat eine Datei gesendet ({file_name}, {file_type}) aber die Daten sind leer angekommen."
|
||||
await self.send_to_core(text, source="app-file")
|
||||
|
||||
elif msg_type == "audio":
|
||||
@@ -1048,26 +1057,33 @@ class ARIABridge:
|
||||
if text.strip():
|
||||
logger.info("[rvs] STT Ergebnis: '%s'", text[:80])
|
||||
# STT-Ergebnis zurueck an die App senden (zur Anzeige, nicht nochmal verarbeiten)
|
||||
await self._send_to_rvs({
|
||||
"type": "stt_result",
|
||||
"payload": {
|
||||
"text": text,
|
||||
"sender": "user",
|
||||
},
|
||||
"timestamp": int(asyncio.get_event_loop().time() * 1000),
|
||||
})
|
||||
try:
|
||||
await self._send_to_rvs({
|
||||
"type": "stt_result",
|
||||
"payload": {
|
||||
"text": text,
|
||||
"sender": "user",
|
||||
},
|
||||
"timestamp": int(asyncio.get_event_loop().time() * 1000),
|
||||
})
|
||||
except Exception as e:
|
||||
logger.warning("[rvs] STT-Ergebnis konnte nicht an App gesendet werden: %s", e)
|
||||
# Text trotzdem an aria-core senden
|
||||
await self.send_to_core(text, source="app-voice")
|
||||
else:
|
||||
logger.info("[rvs] Keine Sprache erkannt — ignoriert")
|
||||
await self._send_to_rvs({
|
||||
"type": "stt_result",
|
||||
"payload": {
|
||||
"text": "",
|
||||
"error": "Keine Sprache erkannt",
|
||||
"sender": "user",
|
||||
},
|
||||
"timestamp": int(asyncio.get_event_loop().time() * 1000),
|
||||
})
|
||||
try:
|
||||
await self._send_to_rvs({
|
||||
"type": "stt_result",
|
||||
"payload": {
|
||||
"text": "",
|
||||
"error": "Keine Sprache erkannt",
|
||||
"sender": "user",
|
||||
},
|
||||
"timestamp": int(asyncio.get_event_loop().time() * 1000),
|
||||
})
|
||||
except Exception as e:
|
||||
logger.warning("[rvs] STT-Fehler konnte nicht an App gesendet werden: %s", e)
|
||||
|
||||
except Exception:
|
||||
logger.exception("[rvs] Audio-Verarbeitung fehlgeschlagen")
|
||||
|
||||
@@ -58,6 +58,7 @@ services:
|
||||
- ./aria-data/ssh:/home/node/.ssh # SSH Keys fuer VM-Zugriff
|
||||
- /tmp/.X11-unix:/tmp/.X11-unix
|
||||
- /var/run/docker.sock:/var/run/docker.sock # VM von innen verwalten
|
||||
- aria-shared:/shared # Shared Volume fuer Datei-Austausch (Bridge <> Core)
|
||||
restart: unless-stopped
|
||||
networks:
|
||||
- aria-net
|
||||
@@ -72,6 +73,7 @@ services:
|
||||
volumes:
|
||||
- ./aria-data/voices:/voices:ro # TTS Stimmen
|
||||
- ./aria-data/config/aria.env:/config/aria.env
|
||||
- aria-shared:/shared # Shared Volume fuer Datei-Austausch (Bridge <> Core)
|
||||
# Audio-Zugriff
|
||||
- /run/user/1000/pulse:/run/user/1000/pulse
|
||||
- /dev/snd:/dev/snd
|
||||
@@ -110,6 +112,7 @@ services:
|
||||
volumes:
|
||||
openclaw-config: # Persistiert ~/.openclaw (Model, Auth, Sessions)
|
||||
claude-config: # Persistiert ~/.claude (Permissions, Settings)
|
||||
aria-shared: # Datei-Austausch zwischen Bridge und Core
|
||||
|
||||
networks:
|
||||
aria-net:
|
||||
|
||||
Reference in New Issue
Block a user