minmal-file-cloud-email-pim.../backend/app/api/backup.py

715 lines
24 KiB
Python

import io
import json
import os
import shutil
import sqlite3
import tempfile
import uuid
import zipfile
from datetime import datetime, timezone
from pathlib import Path
from flask import request, jsonify, current_app, Response
from app.api import api_bp
from app.api.auth import admin_required
from app.extensions import db
# Store active chunked uploads in memory (upload_id -> metadata)
_active_uploads = {}
# --- Backup ---
@api_bp.route('/admin/backup', methods=['POST'])
@admin_required
def create_backup():
"""Create a full backup as streaming ZIP download.
Contains:
- metadata.json (version, timestamp, stats)
- database.sqlite3 (copy of the SQLite DB)
- files/ (all uploaded user files)
"""
db_uri = current_app.config['SQLALCHEMY_DATABASE_URI']
db_path = db_uri.replace('sqlite:///', '')
upload_path = Path(current_app.config['UPLOAD_PATH'])
# Gather stats
from app.models.user import User
from app.models.file import File
user_count = User.query.count()
file_count = File.query.filter_by(is_folder=False).count()
metadata = {
'version': '1.0',
'created_at': datetime.now(timezone.utc).isoformat(),
'user_count': user_count,
'file_count': file_count,
'description': 'Mini-Cloud Full Backup',
}
def generate_zip():
"""Stream ZIP file in chunks."""
# We create the ZIP in a temp file to handle large data
with tempfile.NamedTemporaryFile(delete=False, suffix='.zip') as tmp:
tmp_path = tmp.name
try:
with zipfile.ZipFile(tmp_path, 'w', zipfile.ZIP_DEFLATED,
allowZip64=True) as zf:
# 1. Metadata
zf.writestr('metadata.json', json.dumps(metadata, indent=2))
# 2. SQLite database (safe copy via backup API)
db_backup_path = tmp_path + '.db'
try:
source = sqlite3.connect(db_path)
dest = sqlite3.connect(db_backup_path)
source.backup(dest)
source.close()
dest.close()
zf.write(db_backup_path, 'database.sqlite3')
finally:
if os.path.exists(db_backup_path):
os.unlink(db_backup_path)
# 3. User files
if upload_path.exists():
for file_path in upload_path.rglob('*'):
if file_path.is_file():
arcname = 'files/' + str(file_path.relative_to(upload_path))
zf.write(str(file_path), arcname)
# Stream the ZIP file in 1MB chunks
with open(tmp_path, 'rb') as f:
while True:
chunk = f.read(1024 * 1024)
if not chunk:
break
yield chunk
finally:
if os.path.exists(tmp_path):
os.unlink(tmp_path)
timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
filename = f'minicloud_backup_{timestamp}.zip'
return Response(
generate_zip(),
mimetype='application/zip',
headers={
'Content-Disposition': f'attachment; filename="{filename}"',
'X-Accel-Buffering': 'no',
},
)
# --- Chunked Restore Upload ---
@api_bp.route('/admin/restore/init', methods=['POST'])
@admin_required
def restore_init():
"""Initialize a chunked restore upload.
Returns an upload_id to use for subsequent chunk uploads.
"""
data = request.get_json() or {}
total_size = data.get('total_size', 0)
total_chunks = data.get('total_chunks', 0)
filename = data.get('filename', 'backup.zip')
upload_id = str(uuid.uuid4())
upload_dir = Path(tempfile.gettempdir()) / f'minicloud_restore_{upload_id}'
upload_dir.mkdir(parents=True, exist_ok=True)
_active_uploads[upload_id] = {
'dir': str(upload_dir),
'total_size': total_size,
'total_chunks': total_chunks,
'received_chunks': set(),
'filename': filename,
'created_at': datetime.now(timezone.utc).isoformat(),
}
return jsonify({
'upload_id': upload_id,
'chunk_size': 10 * 1024 * 1024, # 10 MB recommended chunk size
}), 200
@api_bp.route('/admin/restore/chunk', methods=['POST'])
@admin_required
def restore_chunk():
"""Upload a single chunk of the backup file."""
upload_id = request.form.get('upload_id', '')
chunk_number = int(request.form.get('chunk_number', 0))
if upload_id not in _active_uploads:
return jsonify({'error': 'Upload-ID unbekannt. Bitte neu starten.'}), 404
if 'chunk' not in request.files:
return jsonify({'error': 'Kein Chunk gesendet'}), 400
upload_info = _active_uploads[upload_id]
upload_dir = Path(upload_info['dir'])
chunk_file = request.files['chunk']
chunk_path = upload_dir / f'chunk_{chunk_number:06d}'
chunk_file.save(str(chunk_path))
upload_info['received_chunks'].add(chunk_number)
return jsonify({
'chunk_number': chunk_number,
'received': len(upload_info['received_chunks']),
'total': upload_info['total_chunks'],
}), 200
@api_bp.route('/admin/restore/finalize', methods=['POST'])
@admin_required
def restore_finalize():
"""Assemble chunks and perform the restore."""
data = request.get_json() or {}
upload_id = data.get('upload_id', '')
if upload_id not in _active_uploads:
return jsonify({'error': 'Upload-ID unbekannt'}), 404
upload_info = _active_uploads[upload_id]
upload_dir = Path(upload_info['dir'])
try:
# Assemble chunks into ZIP
zip_path = upload_dir / 'backup.zip'
with open(str(zip_path), 'wb') as outfile:
chunk_num = 0
while True:
chunk_path = upload_dir / f'chunk_{chunk_num:06d}'
if not chunk_path.exists():
break
with open(str(chunk_path), 'rb') as cf:
shutil.copyfileobj(cf, outfile)
chunk_num += 1
if chunk_num == 0:
return jsonify({'error': 'Keine Chunks gefunden'}), 400
# Verify it's a valid ZIP
if not zipfile.is_zipfile(str(zip_path)):
return jsonify({'error': 'Ungueltige ZIP-Datei'}), 400
# Perform restore
result = _perform_restore(str(zip_path))
return jsonify(result), 200
except Exception as e:
return jsonify({'error': f'Restore fehlgeschlagen: {str(e)}'}), 500
finally:
# Cleanup
shutil.rmtree(str(upload_dir), ignore_errors=True)
_active_uploads.pop(upload_id, None)
@api_bp.route('/admin/restore/direct', methods=['POST'])
@admin_required
def restore_direct():
"""Direct restore from a small backup file (non-chunked, for files < 500MB)."""
if 'file' not in request.files:
return jsonify({'error': 'Keine Datei gesendet'}), 400
backup_file = request.files['file']
with tempfile.NamedTemporaryFile(delete=False, suffix='.zip') as tmp:
backup_file.save(tmp.name)
tmp_path = tmp.name
try:
if not zipfile.is_zipfile(tmp_path):
return jsonify({'error': 'Ungueltige ZIP-Datei'}), 400
result = _perform_restore(tmp_path)
return jsonify(result), 200
except Exception as e:
return jsonify({'error': f'Restore fehlgeschlagen: {str(e)}'}), 500
finally:
os.unlink(tmp_path)
def _perform_restore(zip_path):
"""Perform the actual restore from a validated ZIP file.
Strategy for DB merge:
- Open backup SQLite as a separate connection
- For each table in the backup, read all rows
- INSERT OR REPLACE into the live database
- This preserves any new tables/columns in the current schema
- Existing data with same primary keys gets overwritten by backup data
"""
upload_path = Path(current_app.config['UPLOAD_PATH'])
stats = {'users': 0, 'files_db': 0, 'files_disk': 0, 'tables': []}
with tempfile.TemporaryDirectory() as extract_dir:
extract_path = Path(extract_dir)
# Extract ZIP
with zipfile.ZipFile(zip_path, 'r') as zf:
zf.extractall(str(extract_path))
# Read metadata
metadata_path = extract_path / 'metadata.json'
metadata = {}
if metadata_path.exists():
metadata = json.loads(metadata_path.read_text())
stats['backup_date'] = metadata.get('created_at', 'Unbekannt')
stats['backup_users'] = metadata.get('user_count', '?')
stats['backup_files'] = metadata.get('file_count', '?')
# Restore database via merge
backup_db_path = extract_path / 'database.sqlite3'
if backup_db_path.exists():
live_db_uri = current_app.config['SQLALCHEMY_DATABASE_URI']
live_db_path = live_db_uri.replace('sqlite:///', '')
backup_conn = sqlite3.connect(str(backup_db_path))
backup_conn.row_factory = sqlite3.Row
live_conn = sqlite3.connect(live_db_path)
try:
# Get list of tables in backup
backup_tables = [row[0] for row in
backup_conn.execute(
"SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'"
).fetchall()]
# Get list of tables in live DB
live_tables = [row[0] for row in
live_conn.execute(
"SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'"
).fetchall()]
for table in backup_tables:
if table == 'alembic_version':
continue
if table not in live_tables:
continue
# Get column names from live table
live_cols = [col[1] for col in
live_conn.execute(f'PRAGMA table_info("{table}")').fetchall()]
backup_cols = [col[1] for col in
backup_conn.execute(f'PRAGMA table_info("{table}")').fetchall()]
# Use only columns that exist in both
common_cols = [c for c in backup_cols if c in live_cols]
if not common_cols:
continue
cols_str = ', '.join(f'"{c}"' for c in common_cols)
placeholders = ', '.join('?' for _ in common_cols)
rows = backup_conn.execute(
f'SELECT {cols_str} FROM "{table}"'
).fetchall()
row_count = 0
for row in rows:
try:
live_conn.execute(
f'INSERT OR REPLACE INTO "{table}" ({cols_str}) VALUES ({placeholders})',
tuple(row)
)
row_count += 1
except Exception:
continue
if row_count > 0:
stats['tables'].append({'name': table, 'rows': row_count})
live_conn.commit()
finally:
backup_conn.close()
live_conn.close()
# Restore files
backup_files_dir = extract_path / 'files'
if backup_files_dir.exists():
upload_path.mkdir(parents=True, exist_ok=True)
file_count = 0
for src_file in backup_files_dir.rglob('*'):
if src_file.is_file():
rel_path = src_file.relative_to(backup_files_dir)
dest = upload_path / rel_path
dest.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(str(src_file), str(dest))
file_count += 1
stats['files_disk'] = file_count
stats['success'] = True
stats['message'] = 'Restore erfolgreich abgeschlossen'
return stats
# ========== Single File Browse & Restore ==========
def _list_zip_contents(zip_path):
"""List files inside a backup ZIP with metadata."""
entries = []
with zipfile.ZipFile(zip_path, 'r') as zf:
for info in zf.infolist():
if info.is_dir():
continue
entries.append({
'path': info.filename,
'size': info.file_size,
'compressed_size': info.compress_size,
'modified': datetime(*info.date_time).isoformat() if info.date_time else None,
})
return entries
def _extract_single_file(zip_path, file_path):
"""Extract a single file from backup ZIP. Returns bytes + filename."""
with zipfile.ZipFile(zip_path, 'r') as zf:
if file_path not in zf.namelist():
return None, None
data = zf.read(file_path)
filename = os.path.basename(file_path)
return data, filename
def _restore_single_file_to_cloud(zip_path, file_path_in_zip):
"""Restore a single file from backup into the live data/files directory."""
upload_path = Path(current_app.config['UPLOAD_PATH'])
if not file_path_in_zip.startswith('files/'):
return False, 'Nur Dateien aus dem files/-Verzeichnis koennen wiederhergestellt werden'
rel_path = file_path_in_zip[len('files/'):]
with zipfile.ZipFile(zip_path, 'r') as zf:
if file_path_in_zip not in zf.namelist():
return False, 'Datei nicht im Backup gefunden'
dest = upload_path / rel_path
dest.parent.mkdir(parents=True, exist_ok=True)
with zf.open(file_path_in_zip) as src, open(str(dest), 'wb') as dst:
shutil.copyfileobj(src, dst)
return True, f'Datei {rel_path} wiederhergestellt'
# --- Local ZIP: browse & single restore ---
@api_bp.route('/admin/restore/browse', methods=['POST'])
@admin_required
def browse_local_backup():
"""Upload a ZIP and list its contents (without restoring)."""
if 'file' not in request.files:
return jsonify({'error': 'Keine Datei gesendet'}), 400
backup_file = request.files['file']
with tempfile.NamedTemporaryFile(delete=False, suffix='.zip') as tmp:
backup_file.save(tmp.name)
tmp_path = tmp.name
try:
if not zipfile.is_zipfile(tmp_path):
os.unlink(tmp_path)
return jsonify({'error': 'Ungueltige ZIP-Datei'}), 400
entries = _list_zip_contents(tmp_path)
# Store path temporarily for subsequent file requests
browse_id = str(uuid.uuid4())
_active_uploads[browse_id] = {
'zip_path': tmp_path,
'created_at': datetime.now(timezone.utc).isoformat(),
'type': 'browse',
}
return jsonify({
'browse_id': browse_id,
'files': entries,
'total': len(entries),
}), 200
except Exception as e:
os.unlink(tmp_path)
return jsonify({'error': str(e)}), 500
@api_bp.route('/admin/restore/browse/<browse_id>/download/<path:file_path>', methods=['GET'])
@admin_required
def download_file_from_backup(browse_id, file_path):
"""Download a single file from a browsed backup ZIP."""
if browse_id not in _active_uploads:
return jsonify({'error': 'Browse-Session abgelaufen'}), 404
info = _active_uploads[browse_id]
zip_path = info.get('zip_path', '')
if not os.path.exists(zip_path):
return jsonify({'error': 'ZIP nicht mehr verfuegbar'}), 404
data, filename = _extract_single_file(zip_path, file_path)
if data is None:
return jsonify({'error': 'Datei nicht gefunden'}), 404
import mimetypes
mime = mimetypes.guess_type(filename)[0] or 'application/octet-stream'
return Response(data, mimetype=mime, headers={
'Content-Disposition': f'attachment; filename="{filename}"',
})
@api_bp.route('/admin/restore/browse/<browse_id>/restore-file', methods=['POST'])
@admin_required
def restore_single_from_local(browse_id):
"""Restore a single file from browsed backup into the live system."""
if browse_id not in _active_uploads:
return jsonify({'error': 'Browse-Session abgelaufen'}), 404
data = request.get_json()
file_path = data.get('file_path', '')
if not file_path:
return jsonify({'error': 'file_path erforderlich'}), 400
info = _active_uploads[browse_id]
zip_path = info.get('zip_path', '')
if not os.path.exists(zip_path):
return jsonify({'error': 'ZIP nicht mehr verfuegbar'}), 404
success, message = _restore_single_file_to_cloud(zip_path, file_path)
if success:
return jsonify({'message': message}), 200
return jsonify({'error': message}), 400
@api_bp.route('/admin/restore/browse/<browse_id>/close', methods=['POST'])
@admin_required
def close_browse_session(browse_id):
"""Clean up a browse session."""
if browse_id in _active_uploads:
info = _active_uploads.pop(browse_id)
zip_path = info.get('zip_path', '')
if zip_path and os.path.exists(zip_path):
os.unlink(zip_path)
return jsonify({'message': 'Session geschlossen'}), 200
# --- SFTP: browse & single restore ---
@api_bp.route('/admin/backup/targets/<int:target_id>/versions/<version_name>/browse', methods=['POST'])
@admin_required
def browse_sftp_version(target_id, version_name):
"""Download a version from SFTP and list its contents."""
from app.models.backup_target import BackupTarget
from app.services.sftp_backup import download_version_from_sftp
target = db.session.get(BackupTarget, target_id)
if not target:
return jsonify({'error': 'Nicht gefunden'}), 404
try:
zip_path = download_version_from_sftp(target, version_name)
entries = _list_zip_contents(zip_path)
browse_id = str(uuid.uuid4())
_active_uploads[browse_id] = {
'zip_path': zip_path,
'created_at': datetime.now(timezone.utc).isoformat(),
'type': 'sftp_browse',
}
return jsonify({
'browse_id': browse_id,
'files': entries,
'total': len(entries),
}), 200
except Exception as e:
return jsonify({'error': str(e)}), 500
# ========== SFTP Backup Targets ==========
@api_bp.route('/admin/backup/targets', methods=['GET'])
@admin_required
def list_backup_targets():
from app.models.backup_target import BackupTarget
targets = BackupTarget.query.order_by(BackupTarget.created_at).all()
return jsonify([t.to_dict() for t in targets]), 200
@api_bp.route('/admin/backup/targets', methods=['POST'])
@admin_required
def create_backup_target():
from app.models.backup_target import BackupTarget
from app.services.crypto_service import encrypt_field
data = request.get_json()
for field in ['name', 'host', 'username']:
if not data.get(field):
return jsonify({'error': f'{field} erforderlich'}), 400
if not data.get('password') and not data.get('private_key'):
return jsonify({'error': 'Passwort oder Private Key erforderlich'}), 400
target = BackupTarget(
name=data['name'],
host=data['host'],
port=data.get('port', 22),
username=data['username'],
remote_path=data.get('remote_path', '/backups/minicloud'),
is_active=data.get('is_active', True),
backup_interval_minutes=data.get('backup_interval_minutes', 1440),
max_versions=data.get('max_versions', 10),
)
if data.get('password'):
target.password_encrypted = encrypt_field(data['password'], 'backup-key')
if data.get('private_key'):
target.private_key_encrypted = encrypt_field(data['private_key'], 'backup-key')
db.session.add(target)
db.session.commit()
return jsonify(target.to_dict()), 201
@api_bp.route('/admin/backup/targets/<int:target_id>', methods=['PUT'])
@admin_required
def update_backup_target(target_id):
from app.models.backup_target import BackupTarget
from app.services.crypto_service import encrypt_field
target = db.session.get(BackupTarget, target_id)
if not target:
return jsonify({'error': 'Nicht gefunden'}), 404
data = request.get_json()
for field in ['name', 'host', 'port', 'username', 'remote_path',
'is_active', 'backup_interval_minutes', 'max_versions']:
if field in data:
setattr(target, field, data[field])
if data.get('password'):
target.password_encrypted = encrypt_field(data['password'], 'backup-key')
if data.get('private_key'):
target.private_key_encrypted = encrypt_field(data['private_key'], 'backup-key')
db.session.commit()
return jsonify(target.to_dict()), 200
@api_bp.route('/admin/backup/targets/<int:target_id>', methods=['DELETE'])
@admin_required
def delete_backup_target(target_id):
from app.models.backup_target import BackupTarget
target = db.session.get(BackupTarget, target_id)
if not target:
return jsonify({'error': 'Nicht gefunden'}), 404
db.session.delete(target)
db.session.commit()
return jsonify({'message': 'Backup-Ziel geloescht'}), 200
@api_bp.route('/admin/backup/targets/<int:target_id>/test', methods=['POST'])
@admin_required
def test_backup_target(target_id):
from app.models.backup_target import BackupTarget
from app.services.sftp_backup import test_sftp_connection
target = db.session.get(BackupTarget, target_id)
if not target:
return jsonify({'error': 'Nicht gefunden'}), 404
try:
test_sftp_connection(target)
return jsonify({'message': 'SFTP-Verbindung erfolgreich'}), 200
except Exception as e:
return jsonify({'error': f'Verbindungsfehler: {str(e)}'}), 400
@api_bp.route('/admin/backup/targets/<int:target_id>/run', methods=['POST'])
@admin_required
def run_backup_now(target_id):
"""Manually trigger a backup to this target."""
from app.models.backup_target import BackupTarget
from app.services.sftp_backup import create_backup_zip, upload_backup_to_sftp
target = db.session.get(BackupTarget, target_id)
if not target:
return jsonify({'error': 'Nicht gefunden'}), 404
db_uri = current_app.config['SQLALCHEMY_DATABASE_URI']
db_path = db_uri.replace('sqlite:///', '')
upload_path = current_app.config['UPLOAD_PATH']
zip_path = None
try:
zip_path = create_backup_zip(db_path, upload_path)
version = upload_backup_to_sftp(target, zip_path, current_app)
target.last_backup_at = datetime.now(timezone.utc)
target.last_backup_status = 'success'
target.last_backup_message = f'Version {version} hochgeladen'
db.session.commit()
return jsonify({'message': f'Backup {version} erfolgreich', 'version': version}), 200
except Exception as e:
target.last_backup_at = datetime.now(timezone.utc)
target.last_backup_status = 'error'
target.last_backup_message = str(e)[:500]
db.session.commit()
return jsonify({'error': f'Backup fehlgeschlagen: {str(e)}'}), 500
finally:
if zip_path and os.path.exists(zip_path):
os.unlink(zip_path)
@api_bp.route('/admin/backup/targets/<int:target_id>/versions', methods=['GET'])
@admin_required
def list_backup_versions(target_id):
from app.models.backup_target import BackupTarget
from app.services.sftp_backup import list_sftp_versions
target = db.session.get(BackupTarget, target_id)
if not target:
return jsonify({'error': 'Nicht gefunden'}), 404
try:
versions = list_sftp_versions(target)
return jsonify(versions), 200
except Exception as e:
return jsonify({'error': str(e)}), 500
@api_bp.route('/admin/backup/targets/<int:target_id>/restore/<version_name>', methods=['POST'])
@admin_required
def restore_from_sftp(target_id, version_name):
"""Download a backup version from SFTP and restore it."""
from app.models.backup_target import BackupTarget
from app.services.sftp_backup import download_version_from_sftp
target = db.session.get(BackupTarget, target_id)
if not target:
return jsonify({'error': 'Nicht gefunden'}), 404
zip_path = None
try:
zip_path = download_version_from_sftp(target, version_name)
result = _perform_restore(zip_path)
return jsonify(result), 200
except Exception as e:
return jsonify({'error': f'Restore fehlgeschlagen: {str(e)}'}), 500
finally:
if zip_path and os.path.exists(zip_path):
os.unlink(zip_path)