Skip to content
Open
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -30,3 +30,18 @@ __pycache__/
Homestead.json
Homestead.yaml
Thumbs.db

# AI tool caches (per-machine)
.agents/
.boost/
.claude/
.codex/
.cursor/
.gemini/
.bak/
.dockerignore.local

_route_refs.txt

.untracked-stash/

Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ public function store(Request $request): RedirectResponse
'description' => ['nullable', 'string', 'max:1000'],
'content' => ['nullable', 'string'],
'enterprise_files' => ['nullable', 'array', 'max:10'],
'enterprise_files.*' => ['file', File::types(['txt', 'md', 'markdown', 'docx'])->max(8 * 1024)],
'enterprise_files.*' => ['file', File::types(['txt', 'md', 'markdown', 'docx', 'pdf', 'ppt', 'pptx'])->max(8 * 1024)],
], [
'enterprise_files.max' => __('admin.enterprise_knowledge.error.files_limit'),
]);
Expand Down
32 changes: 30 additions & 2 deletions app/Http/Controllers/Admin/KnowledgeBaseController.php
Original file line number Diff line number Diff line change
Expand Up @@ -566,9 +566,9 @@ private function validateKnowledgeImportForm(Request $request): array
'risk_level' => ['nullable', 'in:low,medium,high'],
'review_status' => ['nullable', 'in:unreviewed,reviewed'],
'import_action' => ['nullable', 'in:save,save_and_chunk'],
'knowledge_file' => ['nullable', File::types(['txt', 'md', 'docx'])->max(8 * 1024)],
'knowledge_file' => ['nullable', File::types(['txt', 'md', 'docx', 'pdf', 'ppt', 'pptx'])->max(8 * 1024)],
'knowledge_files' => ['nullable', 'array', 'max:10'],
'knowledge_files.*' => ['file', File::types(['txt', 'md', 'docx'])->max(8 * 1024)],
'knowledge_files.*' => ['file', File::types(['txt', 'md', 'docx', 'pdf', 'ppt', 'pptx'])->max(8 * 1024)],
], [
'knowledge_file.mimes' => __('admin.knowledge_bases.error.file_type_invalid'),
'knowledge_file.max' => __('admin.knowledge_bases.error.file_too_large'),
Expand Down Expand Up @@ -1106,6 +1106,34 @@ private function parseUploadedKnowledgeFile(string $absolutePath, string $origin
];
}

if ($extension === 'pdf') {
$content = app(\App\Services\GeoFlow\KnowledgeSourceParser::class)->extractPdfContent($absolutePath);
if ($content === '') {
throw new \RuntimeException(__('admin.knowledge_bases.error.file_type_invalid'));
}

return [
'content' => $content,
'file_type' => 'pdf',
];
}

if ($extension === 'pptx') {
$content = app(\App\Services\GeoFlow\KnowledgeSourceParser::class)->extractPptxContent($absolutePath);
if ($content === '') {
throw new \RuntimeException(__('admin.knowledge_bases.error.file_type_invalid'));
}

return [
'content' => $content,
'file_type' => 'presentation',
];
}

if ($extension === 'ppt') {
throw new \RuntimeException(__('admin.knowledge_bases.error.ppt_legacy_not_supported'));
}

throw new \RuntimeException(__('admin.knowledge_bases.error.file_type_invalid'));
}

Expand Down
169 changes: 168 additions & 1 deletion app/Services/GeoFlow/KnowledgeSourceParser.php
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
use Illuminate\Http\Request;
use Illuminate\Http\UploadedFile;
use Illuminate\Support\Facades\Storage;
use Smalot\PdfParser\Parser as SmalotPdfParser;

final class KnowledgeSourceParser
{
Expand All @@ -14,6 +15,12 @@ final class KnowledgeSourceParser

private const MAX_DOCX_COMPRESSION_RATIO = 100;

private const MAX_PPTX_XML_BYTES = 16 * 1024 * 1024;

private const MAX_PPTX_COMPRESSION_RATIO = 100;

private const MAX_PDF_BYTES = 32 * 1024 * 1024;

public function storeUploadedKnowledgeFile(UploadedFile $file, string $relativeDirectory = 'uploads/knowledge'): string
{
$extension = strtolower($file->getClientOriginalExtension() ?: 'txt');
Expand Down Expand Up @@ -191,7 +198,7 @@ public function resolveKnowledgeFileType(string $requestedType, string $manualCo

$fileType = (string) ($parsedFiles[0]['file_type'] ?? 'markdown');

return in_array($fileType, ['markdown', 'word', 'text'], true) ? $fileType : 'markdown';
return in_array($fileType, ['markdown', 'word', 'text', 'pdf', 'presentation'], true) ? $fileType : 'markdown';
}

/**
Expand Down Expand Up @@ -249,6 +256,34 @@ public function parseUploadedKnowledgeFile(string $absolutePath, string $origina
];
}

if ($extension === 'pdf') {
$content = $this->extractPdfContent($absolutePath);
if ($content === '') {
throw new \RuntimeException(__('admin.knowledge_bases.error.file_type_invalid'));
}

return [
'content' => $content,
'file_type' => 'pdf',
];
}

if ($extension === 'pptx') {
$content = $this->extractPptxContent($absolutePath);
if ($content === '') {
throw new \RuntimeException(__('admin.knowledge_bases.error.file_type_invalid'));
}

return [
'content' => $content,
'file_type' => 'presentation',
];
}

if ($extension === 'ppt') {
throw new \RuntimeException(__('admin.knowledge_bases.error.ppt_legacy_not_supported'));
}

throw new \RuntimeException(__('admin.knowledge_bases.error.file_type_invalid'));
}

Expand Down Expand Up @@ -434,6 +469,138 @@ public function extractDocxContent(string $absolutePath): string
return is_string($content) ? $this->normalizeKnowledgeText($content) : '';
}


public function extractPdfContent(string $absolutePath): string
{
if (@filesize($absolutePath) > self::MAX_PDF_BYTES) {
throw new \RuntimeException(__('admin.knowledge_bases.error.file_too_large'));
}
if (! class_exists(SmalotPdfParser::class)) {
return '';
}

try {
$parser = new SmalotPdfParser;
$document = $parser->parseFile($absolutePath);
$text = $document->getText();
if (! is_string($text) || $text === '') {
return '';
}

if (strlen($text) > self::MAX_KNOWLEDGE_BYTES) {
throw new \RuntimeException(__('admin.knowledge_bases.error.content_too_large'));
}

return $this->normalizeKnowledgeText($this->convertUploadedTextToUtf8($text));
} catch (\Throwable $exception) {
return '';
}
}

public function extractPptxContent(string $absolutePath): string
{
if (! class_exists('ZipArchive') || ! class_exists('XMLReader')) {
return '';
}

$zip = new \ZipArchive;
if ($zip->open($absolutePath) !== true) {
return '';
}

$slideNames = [];
for ($i = 0; $i < $zip->numFiles; $i++) {
$name = $zip->getNameIndex($i);
if (is_string($name) && preg_match('#^ppt/slides/slide\d+\.xml$#', $name)) {
$slideNames[] = $name;
}
}
sort($slideNames, SORT_NATURAL);

if ($slideNames === []) {
$zip->close();
return '';
}

$presentationNamespace = 'http://schemas.openxmlformats.org/drawingml/2006/main';
$textOutput = tmpfile();
if (! is_resource($textOutput)) {
$zip->close();
return '';
}

$slideNumber = 0;
$contentBytes = 0;
foreach ($slideNames as $slideName) {
$stat = $zip->statName($slideName);
$uncompressedSize = is_array($stat) ? max(0, (int) ($stat['size'] ?? 0)) : 0;
$compressedSize = is_array($stat) ? max(1, (int) ($stat['comp_size'] ?? 0)) : 1;
if ($uncompressedSize > self::MAX_PPTX_XML_BYTES || ($uncompressedSize / $compressedSize) > self::MAX_PPTX_COMPRESSION_RATIO) {
fclose($textOutput);
$zip->close();
throw new \RuntimeException(__('admin.knowledge_bases.error.pptx_expansion_too_large'));
}

$source = $zip->getStream($slideName);
if (! is_resource($source)) {
continue;
}

$temporary = tmpfile();
if (! is_resource($temporary)) {
fclose($source);
fclose($textOutput);
$zip->close();
return '';
}

$copiedBytes = stream_copy_to_stream($source, $temporary, self::MAX_PPTX_XML_BYTES + 1);
fclose($source);
if (! is_int($copiedBytes) || $copiedBytes > self::MAX_PPTX_XML_BYTES) {
fclose($temporary);
fclose($textOutput);
$zip->close();
throw new \RuntimeException(__('admin.knowledge_bases.error.pptx_expansion_too_large'));
}

$metadata = stream_get_meta_data($temporary);
$temporaryPath = (string) ($metadata['uri'] ?? '');
$reader = new \XMLReader;
if ($temporaryPath !== '' && @$reader->open($temporaryPath, null, LIBXML_NONET | LIBXML_NOERROR | LIBXML_NOWARNING)) {
$slideNumber++;
fwrite($textOutput, "\n\n# Slide " . $slideNumber . "\n\n");
while ($reader->read()) {
if (
$reader->nodeType === \XMLReader::ELEMENT
&& $reader->localName === 't'
&& $reader->namespaceURI === $presentationNamespace
) {
$value = trim($reader->readString());
if ($value !== '') {
$contentBytes += strlen($value) + 1;
if ($contentBytes > self::MAX_KNOWLEDGE_BYTES) {
$reader->close();
fclose($temporary);
fclose($textOutput);
$zip->close();
throw new \RuntimeException(__('admin.knowledge_bases.error.content_too_large'));
}
fwrite($textOutput, $value . "\n");
}
}
}
$reader->close();
}
fclose($temporary);
}
$zip->close();
rewind($textOutput);
$content = stream_get_contents($textOutput, self::MAX_KNOWLEDGE_BYTES + 1);
fclose($textOutput);

return is_string($content) ? $this->normalizeKnowledgeText($content) : '';
}

private function assertContentSize(string $content): void
{
if (strlen($content) > self::MAX_KNOWLEDGE_BYTES) {
Expand Down
3 changes: 2 additions & 1 deletion composer.json
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,8 @@
"laravel/horizon": "^5.45",
"laravel/reverb": "^1.0",
"laravel/sanctum": "^4.3",
"laravel/tinker": "^2.10.1"
"laravel/tinker": "^2.10.1",
"smalot/pdfparser": "^2.12"
},
"require-dev": {
"fakerphp/faker": "^1.23",
Expand Down
57 changes: 56 additions & 1 deletion composer.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading