How to Integrate Replit with Screaming Frog

1# Python: download crawl CSV from a URL if not uploading directly
2import requests
3import os
4
5# Option A: download from public URL
6CRAWL_URL = os.environ.get('CRAWL_FILE_URL', '')
7if CRAWL_URL:
8    print('Downloading crawl file...')
9    response = requests.get(CRAWL_URL)
10    with open('crawl-export.csv', 'wb') as f:
11        f.write(response.content)
12    print(f'Downloaded: {len(response.content):,} bytes')
13else:
14    # Option B: file already uploaded to Replit
15    if os.path.exists('crawl-export.csv'):
16        size = os.path.getsize('crawl-export.csv')
17        print(f'Using uploaded file: {size:,} bytes')
18    else:
19        print('Error: crawl-export.csv not found. Upload it or set CRAWL_FILE_URL.')

1# Python SEO audit script for Screaming Frog CSV exports
2import pandas as pd
3import sys
4
5def analyze_crawl(filename='crawl-export.csv'):
6    print(f'Loading {filename}...')
7    df = pd.read_csv(filename, encoding='utf-8', low_memory=False)
8
9    # Strip whitespace from column names (Screaming Frog adds some)
10    df.columns = df.columns.str.strip()
11    total_urls = len(df)
12    print(f'Total URLs: {total_urls:,}')
13    print(f'Columns: {list(df.columns[:10])}...\n')
14
15    issues = []
16
17    # 1. Broken links (4xx status codes)
18    if 'Status Code' in df.columns:
19        broken = df[df['Status Code'].between(400, 499)]
20        issues.append({'Issue': '4xx Broken Pages', 'Count': len(broken),
21                       'Pct': f'{len(broken)/total_urls*100:.1f}%',
22                       'Priority': 'Critical'})
23
24    # 2. Server errors (5xx)
25    if 'Status Code' in df.columns:
26        server_errors = df[df['Status Code'].between(500, 599)]
27        issues.append({'Issue': '5xx Server Errors', 'Count': len(server_errors),
28                       'Pct': f'{len(server_errors)/total_urls*100:.1f}%',
29                       'Priority': 'Critical'})
30
31    # 3. Redirects
32    if 'Status Code' in df.columns:
33        redirects = df[df['Status Code'].between(300, 399)]
34        issues.append({'Issue': 'Redirected Pages', 'Count': len(redirects),
35                       'Pct': f'{len(redirects)/total_urls*100:.1f}%',
36                       'Priority': 'High'})
37
38    # 4. Missing meta description
39    for col in ['Meta Description 1', 'Meta Description', 'meta_description']:
40        if col in df.columns:
41            missing_meta = df[df[col].isna() | (df[col].str.strip() == '')]
42            issues.append({'Issue': 'Missing Meta Description', 'Count': len(missing_meta),
43                           'Pct': f'{len(missing_meta)/total_urls*100:.1f}%',
44                           'Priority': 'High'})
45            break
46
47    # 5. Missing or duplicate title tags
48    for col in ['Title 1', 'Title', 'title']:
49        if col in df.columns:
50            missing_title = df[df[col].isna() | (df[col].str.strip() == '')]
51            duplicate_title = df[df[col].duplicated(keep=False) & df[col].notna()]
52            issues.append({'Issue': 'Missing Page Title', 'Count': len(missing_title),
53                           'Pct': f'{len(missing_title)/total_urls*100:.1f}%', 'Priority': 'Critical'})
54            issues.append({'Issue': 'Duplicate Page Titles', 'Count': len(duplicate_title),
55                           'Pct': f'{len(duplicate_title)/total_urls*100:.1f}%', 'Priority': 'High'})
56            break
57
58    # Print summary report
59    print('SEO AUDIT SUMMARY')
60    print('=' * 60)
61    report_df = pd.DataFrame(issues).sort_values('Count', ascending=False)
62    print(report_df.to_string(index=False))
63    print(f'\nReport generated for {total_urls:,} URLs.')
64    return report_df
65
66if __name__ == '__main__':
67    filename = sys.argv[1] if len(sys.argv) > 1 else 'crawl-export.csv'
68    analyze_crawl(filename)

1# Flask web service for Screaming Frog CSV analysis
2import os
3import io
4import pandas as pd
5from flask import Flask, request, jsonify, render_template_string
6
7app = Flask(__name__)
8
9UPLOAD_FORM = '''
10<!DOCTYPE html><html><body>
11<h2>Screaming Frog Crawl Analyzer</h2>
12<form method="post" action="/analyze" enctype="multipart/form-data">
13  <input type="file" name="crawl_file" accept=".csv" required>
14  <button type="submit">Analyze Crawl</button>
15</form>
16</body></html>
17'''
18
19@app.route('/')
20def index():
21    return render_template_string(UPLOAD_FORM)
22
23@app.route('/analyze', methods=['POST'])
24def analyze():
25    if 'crawl_file' not in request.files:
26        return jsonify({'error': 'No file uploaded'}), 400
27
28    file = request.files['crawl_file']
29    if not file.filename.endswith('.csv'):
30        return jsonify({'error': 'Only CSV files supported'}), 400
31
32    content = file.read().decode('utf-8', errors='replace')
33    df = pd.read_csv(io.StringIO(content), low_memory=False)
34    df.columns = df.columns.str.strip()
35
36    total = len(df)
37    results = {'total_urls': total, 'issues': []}
38
39    if 'Status Code' in df.columns:
40        broken = int(df['Status Code'].between(400, 499).sum())
41        results['issues'].append({'category': '4xx Errors', 'count': broken,
42                                  'percent': round(broken/total*100, 1), 'priority': 'critical'})
43        redirects = int(df['Status Code'].between(300, 399).sum())
44        results['issues'].append({'category': 'Redirects', 'count': redirects,
45                                  'percent': round(redirects/total*100, 1), 'priority': 'high'})
46
47    results['issues'].sort(key=lambda x: x['count'], reverse=True)
48    return jsonify(results)
49
50if __name__ == '__main__':
51    app.run(host='0.0.0.0', port=3000)