Move on from android extensions #1653
Description
Describe your suggested feature
Wouldn't it be cool if we could go to Settings > Sources > Add Source and fill a page like this
A source updates it's code? Go to config and update the selectors.
A source now uses cloudflare? Go to config and check "Use Flaresolverr"
We could still have extensions repositories that a JSON with the info you fill in the page above.
Other details
I made this with ChatGPT just to illustrate my request.
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width,initial-scale=1" />
<title>Add New Site — Admin</title>
<style>
:root{--bg:#0f1724;--card:#0b1220;--muted:#94a3b8;--accent:#7c3aed;--success:#16a34a}
*{box-sizing:border-box}
body{font-family:Inter,ui-sans-serif,system-ui,-apple-system,Segoe UI,Roboto,"Helvetica Neue",Arial; margin:0; background:linear-gradient(180deg,#071024 0%, #071629 60%); color:#e6eef8}
.container{max-width:1100px;margin:36px auto;padding:20px}
.card{background:linear-gradient(180deg, rgba(255,255,255,0.02), rgba(255,255,255,0.01)); border:1px solid rgba(255,255,255,0.03); border-radius:12px; padding:20px; box-shadow:0 6px 24px rgba(2,6,23,0.6)}
h1{margin:0 0 8px;font-size:20px}
p.lead{margin:0 0 18px;color:var(--muted)}
.grid{display:grid;gap:12px}
.cols-2{grid-template-columns:1fr 360px}
label{display:block;font-size:13px;color:var(--muted);margin-bottom:6px}
input[type=text], input[type=url], select, textarea{width:100%;padding:10px 12px;border-radius:8px;border:1px solid rgba(255,255,255,0.04);background:transparent;color:inherit;font-size:14px}
textarea{min-height:90px;resize:vertical}
.row{display:flex;gap:10px}
.small{font-size:13px;color:var(--muted)}
.section{margin-top:16px;padding-top:12px;border-top:1px dashed rgba(255,255,255,0.03)}
.selector-row{display:grid;grid-template-columns:120px 1fr 120px 100px;gap:8px;align-items:center;margin-bottom:8px}
.selector-row input[type=text]{padding:8px}
.btn{display:inline-block;padding:8px 12px;border-radius:8px;border:0;background:var(--accent);color:white;font-weight:600;cursor:pointer}
.btn.ghost{background:transparent;border:1px solid rgba(255,255,255,0.06)}
.btn.small{padding:6px 8px;font-size:13px}
.btn.danger{background:#dc2626}
.controls{display:flex;gap:8px;align-items:center}
.right-panel{padding-left:16px}
.preview{background:rgba(255,255,255,0.02);padding:12px;border-radius:8px;border:1px dashed rgba(255,255,255,0.03);font-family:monospace;font-size:13px;color: #cfe6ff;min-height:160px}
.muted-block{background:rgba(255,255,255,0.01);padding:8px;border-radius:8px;border:1px solid rgba(255,255,255,0.02);color:var(--muted);font-size:13px}
.helper{font-size:12px;color:var(--muted);margin-top:6px}
.add-row{display:flex;justify-content:space-between;align-items:center;margin-bottom:6px}
@media (max-width:900px){.cols-2{grid-template-columns:1fr} .right-panel{padding-left:0}}
</style>
</head>
<body>
<div class="container">
<div class="card">
<h1>Add New Site</h1>
<p class="lead">Configure how to extract series, chapters, and pages from a website. You can use CSS selectors, XPath, or JS code (preprocessed).</p>
<div class="grid cols-2">
<div>
<div>
<label>Site Name</label>
<input id="site-name" type="text" placeholder="E.g., MangaExample" />
</div>
<div style="margin-top:12px">
<label>Base URL</label>
<input id="site-base" type="url" placeholder="https://www.example.com" />
<div class="helper">Define the main URL of the site. Relative paths in selectors will be resolved relative to this.</div>
</div>
<div class="section">
<label>Fetch Method</label>
<div class="row" style="margin-bottom:10px">
<select id="fetch-method">
<option value="html">HTML (GET)</option>
<option value="post">HTML (POST)</option>
<option value="api">API (JSON/XML)</option>
</select>
<label style="display:flex;align-items:center;gap:8px;margin-left:8px"><input id="requires-js" type="checkbox"/> Dynamic JS Site (use browser/headless)</label>
</div>
<div class="helper">If you check "Dynamic JS Site," the system will attempt to use the headless engine (Selenium/Panther).</div>
</div>
<div class="section">
<h3 style="margin:0 0 8px">Main Selectors</h3>
<div class="muted-block">Specify for each type the <strong>type</strong> of extractor (css/xpath/js) and the <strong>selector</strong> or code. For JS, you can return a JSON object with the information.</div>
<div style="margin-top:10px">
<label><strong>Series</strong> List Selector</label>
<div class="selector-row">
<select data-key="series.type">
<option value="css">CSS</option>
<option value="xpath">XPath</option>
<option value="js">JS</option>
</select>
<input data-key="series.selector" type="text" placeholder="E.g., .list .comic-item a" />
<input data-key="series.title" type="text" placeholder="Title selector (e.g., .title)" />
<button class="btn small" data-action="add-sample" data-target="series">+</button>
</div>
<div class="selector-row">
<select data-key="series.titleType">
<option value="css">CSS</option>
<option value="xpath">XPath</option>
<option value="attr">Attribute (href, title)</option>
</select>
<input data-key="series.titleSel" type="text" placeholder="E.g., .title, a[title], or @title" />
<input data-key="series.coverSel" type="text" placeholder="Cover selector (e.g., img.cover)" />
<button class="btn small ghost" data-action="hint" data-target="series">?</button>
</div>
</div>
<div style="margin-top:12px">
<label><strong>Chapters</strong> Selector</label>
<div class="selector-row">
<select data-key="chapters.type">
<option value="css">CSS</option>
<option value="xpath">XPath</option>
<option value="js">JS</option>
</select>
<input data-key="chapters.selector" type="text" placeholder="E.g., .chapter-list a" />
<input data-key="chapters.title" type="text" placeholder="Chapter title selector" />
<button class="btn small" data-action="add-sample" data-target="chapters">+</button>
</div>
<div class="selector-row">
<select data-key="chapters.dateType">
<option value="css">CSS</option>
<option value="xpath">XPath</option>
<option value="meta">Meta (data-attr)</option>
</select>
<input data-key="chapters.dateSel" type="text" placeholder="E.g., .date or @data-date" />
<input data-key="chapters.urlAttr" type="text" placeholder="URL attribute (e.g., href)" />
<button class="btn small ghost" data-action="hint" data-target="chapters">?</button>
</div>
</div>
<div style="margin-top:12px">
<label><strong>Pages / Images</strong> Selector</label>
<div class="selector-row">
<select data-key="pages.type">
<option value="css">CSS</option>
<option value="xpath">XPath</option>
<option value="js">JS</option>
</select>
<input data-key="pages.selector" type="text" placeholder="E.g., .page-content img" />
<input data-key="pages.attr" type="text" placeholder="Image attribute (e.g., src, data-src)" />
<button class="btn small" data-action="add-sample" data-target="pages">+</button>
</div>
</div>
<div class="helper">You can add multiple rules per type (e.g., fallback selectors). Use the + buttons to add new ones.</div>
</div>
<div class="section">
<h3 style="margin:0 0 8px">Advanced Options</h3>
<label>Preprocessing (JS)</label>
<textarea id="preprocess-js" placeholder="// Optional: function that receives 'document' and returns an object with extracted fields\n// return { title: ..., chapters: [...] }"></textarea>
<div style="margin-top:10px" class="row">
<label style="display:flex;align-items:center;gap:8px"><input id="use-proxy" type="checkbox"/> Use proxy/CORS (if necessary)</label>
<label style="display:flex;align-items:center;gap:8px;margin-left:8px"><input id="enabled" type="checkbox" checked/> Site enabled</label>
</div>
</div>
<div style="margin-top:14px;display:flex;gap:8px;align-items:center">
<button id="save-btn" class="btn">Save Site</button>
<button id="test-btn" class="btn ghost">Test Selectors</button>
<button id="reset-btn" class="btn small ghost">Clear</button>
</div>
</div>
<aside class="right-panel">
<div style="display:flex;gap:8px;align-items:center;justify-content:space-between;margin-bottom:8px">
<div>
<label>JSON Preview</label>
<div class="helper">Shows the object that will be sent to the backend.</div>
</div>
<div class="controls"><button id="copy-json" class="btn small ghost">Copy</button></div>
</div>
<pre id="json-preview" class="preview">{
"site": "...",
"base": "...",
"selectors": { }
}</pre>
<div style="margin-top:12px">
<label>Notes / Instructions</label>
<div class="muted-block">You can add multiple rules for each type. When a rule fails, the crawler will try the next one (fallback). For sites with dynamic JS, check the "Dynamic JS Site" box; the system will use Selenium/Headless.</div>
</div>
<div style="margin-top:12px">
<label>Test Logs</label>
<div id="test-log" class="muted-block" style="min-height:80px;white-space:pre-wrap;font-family:monospace;color:#d1f2d8"></div>
</div>
</aside>
</div>
</div>
</div>
<script>
// UI helpers: build JSON and handle rows
function q(selector, root=document){return root.querySelector(selector)}
function qs(selector, root=document){return Array.from(root.querySelectorAll(selector))}
function buildPayload(){
const site = document.getElementById('site-name').value.trim();
const base = document.getElementById('site-base').value.trim();
const method = document.getElementById('fetch-method').value;
const requiresJs = document.getElementById('requires-js').checked;
const preprocess = document.getElementById('preprocess-js').value;
const selectors = {};
// gather all elements with data-key attributes
document.querySelectorAll('[data-key]').forEach(el=>{
const key = el.getAttribute('data-key');
const val = el.value;
// key like series.selector or chapters.type
const parts = key.split('.');
let cur = selectors;
for(let i=0;i<parts.length;i++){
if(i===parts.length-1){ cur[parts[i]] = val; } else { cur[parts[i]] = cur[parts[i]] || {}; cur = cur[parts[i]] }
}
})
const payload = { site, base, method, requiresJs, preprocess, selectors };
return payload;
}
function renderPreview(){
const p = buildPayload();
q('#json-preview').textContent = JSON.stringify(p, null, 2);
}
document.addEventListener('input', renderPreview);
document.addEventListener('change', renderPreview);
renderPreview();
// Buttons
document.getElementById('save-btn').addEventListener('click', async ()=>{
const payload = buildPayload();
// Example: POST to the backend. Implement /api/sites on your server.
try{
const res = await fetch('/api/sites', {method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify(payload)});
if(res.ok){ alert('Site saved'); } else { alert('Error saving (see console)'); console.error(await res.text()) }
} catch(e){ alert('Could not connect to the server. Implement /api/sites'); console.error(e) }
});
document.getElementById('reset-btn').addEventListener('click', ()=>{ if(confirm('Clear the form?')) location.reload()});
document.getElementById('copy-json').addEventListener('click', ()=>{ navigator.clipboard.writeText(q('#json-preview').textContent).then(()=>{ alert('JSON copied') }) });
document.getElementById('test-btn').addEventListener('click', async ()=>{
const payload = buildPayload();
const log = q('#test-log'); log.textContent = 'Starting test...';
// Note: CORS in the browser prevents direct fetch to other domains.
log.textContent += '\nSend this JSON to your test endpoint on the server to perform actual scraping.';
log.textContent += '\nJSON:\n' + JSON.stringify(payload, null, 2);
});
// Small helper: add sample selector when + is clicked
document.querySelectorAll('[data-action="add-sample"]').forEach(btn=>{
btn.addEventListener('click', (e)=>{
const target = btn.getAttribute('data-target');
// add a new pair of inputs for that target (simple clone)
const row = document.createElement('div');
row.className = 'selector-row';
row.innerHTML = `
<select data-key="${target}.type"><option value="css">CSS</option><option value="xpath">XPath</option><option value="js">JS</option></select>
<input data-key="${target}.selector" type="text" placeholder="Additional selector (fallback)" />
<input data-key="${target}.extra" type="text" placeholder="Optional (attribute / title)" />
<button class="btn small ghost" data-action="remove">Remove</button>
`;
btn.closest('.section').appendChild(row);
row.querySelector('[data-action="remove"]').addEventListener('click', ()=>{row.remove(); renderPreview()});
renderPreview();
})
})
// hint buttons
document.querySelectorAll('[data-action="hint"]').forEach(btn=>btn.addEventListener('click', ()=>{
alert('Tip: use @href or @data-src to indicate attributes. For JS, write a function that receives `document` and returns the expected fields.');
}))
</script>
</body>
</html>Acknowledgements
- I have searched the existing issues and this is a new ticket, NOT a duplicate or related to another open or closed issue.
- I have written a short but informative title (ideally less than ~100 characters).
- I have updated to the latest version.
- I have filled out all of the requested information in this form, including specific version numbers.