feat: fetch for the assistant with Поиск on - a page's main text for the model, the server's own network out of reach; shorter words in Настройки and the chat; the + menu's roots a step left. Also files from panes into the chat, docview and drag fixes (by the user)
This commit is contained in:
redtea committed
1 parent
84b42c5369
commit
ae8fffe7d8
17 files changed
+712
-102
No files matched your search
@@ -524,5 +524,152 @@
|
||||
return sheets(list);
|
||||
}
|
||||
|
||||
window.DocView = {docx, xlsx, ods, odt, csv, doc, sqlite, bindTabs};
|
||||
// Extract text from PDF content streams without a heavyweight renderer or OCR.
|
||||
// Flate-compressed page streams are inflated with the browser's native API.
|
||||
async function pdf(buf) {
|
||||
const input = new Uint8Array(buf);
|
||||
let source = '';
|
||||
for (let i = 0; i < input.length; i += 0x8000) source += String.fromCharCode(...input.subarray(i, i + 0x8000));
|
||||
if (!source.startsWith('%PDF-')) throw new Error('это не PDF-файл');
|
||||
const objects = /\b(\d+)\s+\d+\s+obj\b/g, pages = [];
|
||||
let inflatedBytes = 0, textBytes = 0, match;
|
||||
async function inflate(data) {
|
||||
const reader = new Blob([data]).stream().pipeThrough(new DecompressionStream('deflate')).getReader();
|
||||
const chunks = []; let total = 0;
|
||||
for (;;) {
|
||||
const part = await reader.read();
|
||||
if (part.done) break;
|
||||
total += part.value.length;
|
||||
if (total > 16 * 1024 * 1024 || inflatedBytes + total > 32 * 1024 * 1024) {
|
||||
await reader.cancel(); throw new Error('PDF stream слишком велик');
|
||||
}
|
||||
chunks.push(part.value);
|
||||
}
|
||||
const result = new Uint8Array(total); let at = 0;
|
||||
chunks.forEach(chunk => { result.set(chunk, at); at += chunk.length; });
|
||||
inflatedBytes += total;
|
||||
return result;
|
||||
}
|
||||
while ((match = objects.exec(source))) {
|
||||
const bodyStart = objects.lastIndex, possibleEnd = source.indexOf('endobj', bodyStart);
|
||||
if (possibleEnd < 0) break;
|
||||
const probe = source.slice(bodyStart, possibleEnd), stream = /\bstream\r?\n/.exec(probe);
|
||||
if (!stream) { objects.lastIndex = possibleEnd + 6; continue; }
|
||||
const dict = probe.slice(0, stream.index);
|
||||
let start = bodyStart + stream.index + stream[0].length;
|
||||
const length = /\/Length\s+(\d+)\b/.exec(dict);
|
||||
let end = length ? start + Number(length[1]) : source.indexOf('endstream', start);
|
||||
if (end < start) { objects.lastIndex = possibleEnd + 6; continue; }
|
||||
let endObj = source.indexOf('endobj', length ? end : end + 9);
|
||||
if (endObj < 0) break;
|
||||
if (/\/Subtype\s*\/Image|\/Type\s*\/(?:ObjStm|XObject|Font|FontFile)/.test(dict)) { objects.lastIndex = endObj + 6; continue; }
|
||||
let raw = source.slice(start, end);
|
||||
if (!length) raw = raw.replace(/[\r\n]+$/, '');
|
||||
let data = Uint8Array.from(raw, c => c.charCodeAt(0) & 255);
|
||||
if (dict.includes('/FlateDecode') && !dict.includes('/ASCII85Decode')) {
|
||||
try { data = await inflate(data); }
|
||||
catch (e) { objects.lastIndex = endObj + 6; continue; }
|
||||
} else if (/\/Filter\b/.test(dict)) { objects.lastIndex = endObj + 6; continue; }
|
||||
let content = '';
|
||||
for (let i = 0; i < data.length; i += 0x8000) content += String.fromCharCode(...data.subarray(i, i + 0x8000));
|
||||
if (content.includes('BT') && (content.includes('Tj') || content.includes('TJ'))) {
|
||||
pages.push(content);
|
||||
textBytes += content.length;
|
||||
if (textBytes >= 2 * 1024 * 1024) break;
|
||||
}
|
||||
objects.lastIndex = endObj + 6;
|
||||
}
|
||||
const decode = bytes => {
|
||||
if (bytes.length > 1 && bytes[0] === 0xfe && bytes[1] === 0xff) {
|
||||
const swapped = new Uint8Array(bytes.length - 2);
|
||||
for (let i = 2; i + 1 < bytes.length; i += 2) { swapped[i - 2] = bytes[i + 1]; swapped[i - 1] = bytes[i]; }
|
||||
return new TextDecoder('utf-16le').decode(swapped);
|
||||
}
|
||||
return new TextDecoder('windows-1252').decode(bytes);
|
||||
};
|
||||
function readString(s, at, literal) {
|
||||
const bytes = [];
|
||||
if (literal) {
|
||||
let depth = 1, i = at + 1;
|
||||
for (; i < s.length && depth; i++) {
|
||||
let c = s.charCodeAt(i);
|
||||
if (c === 92) {
|
||||
c = s.charCodeAt(++i);
|
||||
if (c === 110) bytes.push(10); else if (c === 114) bytes.push(13); else if (c === 116) bytes.push(9);
|
||||
else if (c === 98) bytes.push(8); else if (c === 102) bytes.push(12);
|
||||
else if (c === 10) { /* escaped line break */ }
|
||||
else if (c === 13) { if (s.charCodeAt(i + 1) === 10) i++; }
|
||||
else if (c >= 48 && c <= 55) {
|
||||
let oct = String.fromCharCode(c);
|
||||
for (let n = 0; n < 2 && /[0-7]/.test(s.charAt(i + 1)); n++) oct += s.charAt(++i);
|
||||
bytes.push(parseInt(oct, 8) & 255);
|
||||
} else bytes.push(c & 255);
|
||||
} else if (c === 40) { depth++; bytes.push(c); }
|
||||
else if (c === 41) { if (--depth) bytes.push(c); }
|
||||
else bytes.push(c & 255);
|
||||
}
|
||||
return {value: decode(Uint8Array.from(bytes)), end: i};
|
||||
}
|
||||
let hex = '', i = at + 1;
|
||||
for (; i < s.length && s.charAt(i) !== '>'; i++) if (!/\s/.test(s.charAt(i))) hex += s.charAt(i);
|
||||
if (hex.length % 2) hex += '0';
|
||||
for (let i = 0; i + 1 < hex.length; i += 2) bytes.push(parseInt(hex.slice(i, i + 2), 16));
|
||||
return {value: decode(Uint8Array.from(bytes)), end: i + 1};
|
||||
}
|
||||
function textFrom(content) {
|
||||
let out = '', inText = false, i = 0, operands = [];
|
||||
const token = () => {
|
||||
while (i < content.length && /[\x00\t\n\f\r ]/.test(content.charAt(i))) i++;
|
||||
if (content.charAt(i) === '%') { while (i < content.length && content.charAt(i) !== '\n') i++; return token(); }
|
||||
if (i >= content.length) return null;
|
||||
const c = content.charAt(i);
|
||||
if (c === '(') { const v = readString(content, i, true); i = v.end; return {type: 'str', value: v.value}; }
|
||||
if (c === '<' && content.charAt(i + 1) !== '<') { const v = readString(content, i, false); i = v.end; return {type: 'str', value: v.value}; }
|
||||
if (c === '[') {
|
||||
i++; const values = [];
|
||||
while (i < content.length) {
|
||||
while (/[\x00\t\n\f\r ]/.test(content.charAt(i))) i++;
|
||||
if (content.charAt(i) === ']') { i++; break; }
|
||||
if (content.charAt(i) === '(' || (content.charAt(i) === '<' && content.charAt(i + 1) !== '<')) {
|
||||
const v = readString(content, i, content.charAt(i) === '('); i = v.end; values.push(v.value);
|
||||
} else {
|
||||
const number = /^[+-]?(?:\d+\.?\d*|\.\d+)/.exec(content.slice(i));
|
||||
if (number) { values.push(Number(number[0])); i += number[0].length; } else i++;
|
||||
}
|
||||
}
|
||||
return {type: 'array', value: values};
|
||||
}
|
||||
if (c === '/') { i++; while (i < content.length && !/[\x00\t\n\f\r ()<>\[\]{}\/%]/.test(content.charAt(i))) i++; return {type: 'name'}; }
|
||||
const start = i++;
|
||||
while (i < content.length && !/[\x00\t\n\f\r ()<>\[\]{}\/%]/.test(content.charAt(i))) i++;
|
||||
return {type: 'word', value: content.slice(start, i)};
|
||||
};
|
||||
while (i < content.length) {
|
||||
const t = token();
|
||||
if (!t) break;
|
||||
if (t.type !== 'word' || !/^[A-Za-z*'"]+$/.test(t.value)) { operands.push(t); continue; }
|
||||
const op = t.value;
|
||||
if (op === 'BT') { inText = true; operands = []; continue; }
|
||||
if (op === 'ET') { inText = false; out += '\n'; operands = []; continue; }
|
||||
if (inText && ['Tj', 'TJ', "'", '"'].includes(op)) {
|
||||
if (op === "'" || op === '"') out += '\n';
|
||||
const v = operands[operands.length - 1];
|
||||
if (v && v.type === 'str') out += v.value;
|
||||
else if (v && v.type === 'array') out += v.value.map(x => typeof x === 'number' ? (x < -120 ? ' ' : '') : x).join('');
|
||||
} else if (inText && op === 'Tm') {
|
||||
if (out && !out.endsWith('\n')) out += '\n';
|
||||
} else if (inText && (op === 'T*' || op === 'Td' || op === 'TD')) {
|
||||
if (op === 'T*' || Number(operands[operands.length - 1]?.value) !== 0) out += '\n';
|
||||
else out += ' ';
|
||||
}
|
||||
operands = [];
|
||||
}
|
||||
return out;
|
||||
}
|
||||
const text = pages.map(textFrom).join('\n').replace(/[\t ]+\n/g, '\n').replace(/ {2,}/g, ' ').replace(/\n{3,}/g, '\n\n').trim();
|
||||
if (!text) throw new Error('в PDF не найден извлекаемый текст (сканированные страницы без OCR не читаются)');
|
||||
return text.slice(0, 120000);
|
||||
}
|
||||
|
||||
window.DocView = {docx, xlsx, ods, odt, csv, doc, pdf, sqlite, bindTabs};
|
||||
})();
|
||||
Reference in new issue
Block a user