iang
/
ragger
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

multi types of text support

This commit is contained in:
Ian Griffin 2024-10-28 14:24:09 +08:00
parent 7cbc71d506
commit 6a6b41250e
1 changed files with 4 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
ragger.py
View File

@ -7,7 +7,6 @@
# You should have received a copy of the GNU General Public License along with this program. If not, see <https://www.gnu.org/licenses/>. # You should have received a copy of the GNU General Public License along with this program. If not, see <https://www.gnu.org/licenses/>.
import os import os
import marko
import mimetypes import mimetypes
import re import re
@ -59,7 +58,7 @@ def main():
# #
loaders = { loaders = {
"text/plain": lambda file: TextLoader(file).load(), "text": lambda file: TextLoader(file).load(),
"application/pdf": lambda file: PyPDFLoader(file).load(), "application/pdf": lambda file: PyPDFLoader(file).load(),
"url": lambda file: WebBaseLoader(file).load(), "url": lambda file: WebBaseLoader(file).load(),
} }
@ -81,6 +80,9 @@ def main():
# detect filetype # detect filetype
else: else:
mimetype, _ = mimetypes.guess_type(path) mimetype, _ = mimetypes.guess_type(path)
if mimetype.startswith("text/"):
mimetype = "text"
if mimetype not in loaders: if mimetype not in loaders:
raise ValueError("Unsupported file type: %s" % mimetype) raise ValueError("Unsupported file type: %s" % mimetype)
else: else: