o Mdh.@sddlZddlmZddlmZddlmZmZmZddl m Z ddl m Z ddl mZddlZddlZddlmZddlZddlZeeZGd d d eZdS) N) BaseCommand)default_storage)ChatbotFile RAGMetadata) PdfReader)Document)SentenceTransformer)settingsc@s\eZdZdZddZddZddZdd Zd d ZdddZ dddZ ddZ ddZ dS)CommandzTProcesses PDFs, DOCX, and TXT files for a chatbot and creates RAG data with chunkingcCs*|jdtdd|jddgddddS) N chatbot_idzID of the chatbot to process)typehelpz--chunk_strategysentence)rfixed paragraphzVChunking strategy: sentence (with paragraph awareness), fixed-size, or paragraph-based)defaultchoicesr) add_argumentint)selfparserrO/var/www/ai-chatbot-integration/chatbot/management/commands/process_rag_data.py add_argumentss  zCommand.add_argumentsc CszGt|d7}t|}d}t|jdD]\}}|pd}||d7}|jd|dt|dq|WdWS1s@wYWdStyW}ztd d}~ww) z/Extract text from PDF with paragraph detection.rb zExtracted text from page z ( characters)NzFailed to extract text from PDF) openr enumeratepages extract_textstdoutwritelen Exception) r file_pathfpdftextpage_numpage page_texterrrextract_text_from_pdfs    (zCommand.extract_text_from_pdfc Cs~z)t|}d}|jD]}|jr||jd7}q |jdt|d|WSty>}z tdt|d}~ww)z0Extract text from DOCX with paragraph detection.rrzExtracted text from DOCX (rz"Failed to extract text from DOCX: N) r paragraphsr+stripr$r%r&r'str)rr(docr+parar/rrrextract_text_from_docx(s  zCommand.extract_text_from_docxc Csz:t|ddd(}|}ddd|dD}|jdt|d|Wd WS1s3wYWd StyO}z td t|d }~ww) zExtract text from TXT file.rzutf-8)encodingrcss |] }|r|VqdS)Nr2).0r5rrr :sz0Command.extract_text_from_txt..zExtracted text from TXT (rNz!Failed to extract text from TXT: ) r readjoinsplitr$r%r&r'r3)rr(r)r+r/rrrextract_text_from_txt5s(zCommand.extract_text_from_txtcCsXtj|d}|dkr||S|dkr||S|dkr%||Std|)z0Extract text from a file based on its extension.rz.pdfz.docxz.txtzUnsupported file type: )ospathsplitextlowerr0r6r? ValueError)rr(file_extensionrrrextract_text_from_file@s   zCommand.extract_text_from_filePcCstd}dd|dD}g}d}|jdt|d|D]}||} g} d} dd| jD} | D]k} t|| }| ||kr| rz|d | |d |d |d 7}|rfd | | d dnd}|rn|| gn| g} t|||} q5| | |d | |d |d |d 7}g} d} q5| | | |7} q5| r|d | |d |d |d 7}q!|jdt|d|S)z@Chunk text into sentences while respecting paragraph boundaries.en_core_web_smcSg|] }|r|qSrr9r:prrr Oz-Command.sentence_chunking..rr Detected paragraphscSs g|] }|jr|jqSr)r+r2)r:sentrrrrMY  chunk_r+indexidr NrCreated z sentence-based chunks) spacyloadr>r$r%r&sentsappendr=)rr+ max_tokensoverlap_tokensnlpr1chunks chunk_indexr5r4 current_chunkcurrent_tokens sentencesrQ sent_tokens overlap_textrrrsentence_chunkingLsX       zCommand.sentence_chunkingcs2fddtdtD}ddt|DS)zFallback fixed-size chunking.csg|] }||qSrr)r:i chunk_sizer+rrrMrNz*Command.fixed_chunking..rcSs"g|] \}}||d|dqS)rTrUr)r:rjchunkrrrrMs  )ranger&r!)rr+rlrarrkrfixed_chunkings zCommand.fixed_chunkingcCszdd|dD}g}d}|jdt|d|D]}|||d|d|d 7}q|jd t|d |S) zChunk text into paragraphs.cSrJrr9rKrrrrMrNz.Command.paragraph_chunking..rrrOrPrTrUrrYz paragraph-based chunks)r>r$r%r&r])rr+r1rarbr5rrrparagraph_chunkings zCommand.paragraph_chunkingc s`t}|d|d}ztjjd}|jjd|jdddd|jWn tjyJ|jj|j d d dd|jYdSwt jj |d }|sk|jj|j d |jdd|jdStjttjd d}d}z|j|d|jjd|dd|jWn|jjd|dd|jYz|j|d} |jjd|dd|jWn)ty} z|jj|j dt| dd|jWYd} ~ dSd} ~ ww|jjddd|jtd} |jjddd|j|D]|jjdjjdd|jt} tjjddid\} }ztjj}|jjd|dd|j||}|jjdt|ddd|j|jjd|d dd|j|d!kr|j|d"d#d$}n|d%kr|j|d&d'}n |d(kr| |}|jjd)t|d*dd|j|jjd+dd|jd,d-|D}| j!|d.d/}|jjd0t|d1dd|jd2d-|D}fd3d-|D}| j"|||fd4d-|Dd5|jjd6t|d7dd|jt|| _#d8| _$| %|jjd9jjdd|jt| }|jjd:|d;d<dd|jWq ty} z=d=| _$t| | _&| %|jj|j d>jjd?t| dd|jt'(d>jjd?t| WYd} ~ q d} ~ wwt|}|jj|j )d@|jdA|d;d<dd|jdS)BNr chunk_strategyrWz%Starting RAG processing for chatbot: z (ID: ) )endingzChatbot with ID z does not exist)chatbotzNo files found for chatbot chroma_db)rAchatbot_)namezCleared existing collection: z!No existing collection to clear: zInitialized Chroma collection: z(Failed to initialize Chroma collection: z&Loading sentence-transformers model...zall-MiniLM-L6-v2z#Embedding model loaded successfullyz Processing file: status PROCESSING)filedefaultszReading file from: zTotal text extracted: z characterszChunking text using z strategy...rrGrH)r^r_rri)rlrrYz chunkszGenerating embeddings...cSg|]}|dqS)r+rr:rmrrrrMz"Command.handle..F)show_progress_barz Generated z embeddingscSr~rrrrrrrrMrcs$g|]}j|d|ddqS)rVr+)r file_idrbr+rrrr file_objrrrMs cs g|] }jd|dqS)_rVrrr)rrrrMrR) documents embeddings metadatasidszStored z chunks in Chroma collection COMPLETEDzUpdated RAG metadata for file: zFile processing completed in z.2fz secondsFAILEDzFailed to process file z: z& RAG processing completed for chatbot z in )*timerobjectsgetr$r%ryflush DoesNotExiststyleERRORrfilterWARNINGchromadbPersistentClientr3r MEDIA_ROOTdelete_collectionget_or_create_collectionr'r r|r get_or_createrrArFr&rhrorpencodeadd chunk_countrzsave error_messageloggererrorSUCCESS)rargsoptions start_timerqrvfiles chroma_clientcollection_name collectionr/embedderfile_start_time rag_metadatacreatedr(r+ra chunk_textsrrr file_durationtotal_durationrrrhandles         "                       ,  *zCommand.handleN)rGrH)ri) __name__ __module__ __qualname__rrr0r6r?rFrhrorprrrrrr s  5 r )rdjango.core.management.baserdjango.core.files.storagerchatbot.modelsrrrPyPDF2rdocxrsentence_transformersr rr@ django.confr rZlogging getLoggerrrr rrrrs