Skip to content

Commit

Permalink
bump
Browse files Browse the repository at this point in the history
  • Loading branch information
jph00 committed Sep 1, 2024
1 parent e91f8cc commit 7bd2a1a
Show file tree
Hide file tree
Showing 3 changed files with 27 additions and 35 deletions.
2 changes: 1 addition & 1 deletion llms_txt/__init__.py
Original file line number Diff line number Diff line change
@@ -1 +1 @@
__version__ = "0.0.1"
__version__ = "0.0.2"
26 changes: 13 additions & 13 deletions llms_txt/core.py
Original file line number Diff line number Diff line change
@@ -1,20 +1,20 @@
"""Helpers to create and use llms.txt files"""

# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/00_core.ipynb.
# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/01_core.ipynb.

# %% auto 0
__all__ = ['Sections', 'Project', 'parse_llms_file', 'Doc', 'Section', 'mk_ctx', 'get_sizes', 'llms_txt2ctx']

# %% ../nbs/00_core.ipynb
# %% ../nbs/01_core.ipynb
import re

# %% ../nbs/00_core.ipynb
# %% ../nbs/01_core.ipynb
from fastcore.utils import *
from fastcore.xml import *
from fastcore.script import *
import httpx

# %% ../nbs/00_core.ipynb
# %% ../nbs/01_core.ipynb
def _opt_re(s): return f'(?:{s})?'

def _parse_llms_txt(txt):
Expand All @@ -24,20 +24,20 @@ def _parse_llms_txt(txt):
match = re.search(pat, txt, flags=(re.DOTALL | re.MULTILINE))
return match.groupdict() if match else None

# %% ../nbs/00_core.ipynb
# %% ../nbs/01_core.ipynb
def _split_on_h2(text):
parts = re.split(r'\n?## ', text)
details = parts[0].strip() if parts[0].strip() else None
sections = [f"## {p.strip()}" for p in parts[1:] if p.strip()]
return details, sections

# %% ../nbs/00_core.ipynb
# %% ../nbs/01_core.ipynb
def _parse_section(section):
title = section.split('\n', 1)[0].strip('# ')
links = re.findall(r'\[(.+?)\]\((.+?)\)(?:: (.+?))?(?=\n|$)', section)
return title, [(t, u, d.strip() if d else None) for t, u, d in links]

# %% ../nbs/00_core.ipynb
# %% ../nbs/01_core.ipynb
def parse_llms_file(txt):
parsed = _parse_llms_txt(txt)
if not parsed: return None
Expand All @@ -46,35 +46,35 @@ def parse_llms_file(txt):
del parsed['rest']
return dict2obj(parsed)

# %% ../nbs/00_core.ipynb
# %% ../nbs/01_core.ipynb
Sections = partial(ft, 'sections')
Project = partial(ft, 'project')

# %% ../nbs/00_core.ipynb
# %% ../nbs/01_core.ipynb
def Doc(url, **kw):
"Create a `Doc` FT object with the text retrieved from `url` as the child, and `kw` as attrs."
re_comment = re.compile('^<!--.*-->$', flags=re.MULTILINE)
txt = [o for o in httpx.get(url).text.splitlines() if not re_comment.search(o)]
return ft('doc', '\n'.join(txt), **kw)

# %% ../nbs/00_core.ipynb
# %% ../nbs/01_core.ipynb
def Section(nm, items):
"Create a `Section` FT object containing a `Doc` object for each child."
return ft(nm, *[Doc(title=title, url=url, detl=detl) for title,url,detl in items])

# %% ../nbs/00_core.ipynb
# %% ../nbs/01_core.ipynb
def mk_ctx(d, optional=True):
"Create a `Project` with a `Section` for each H2 part in `d`, optionally skipping the 'optional' section."
skip = '' if optional else 'Optional'
sections = [Section(k, v) for k,v in d.sections.items() if k!=skip]
return Project(title=d.title, summary=d.summary, details=d.details)(*sections)

# %% ../nbs/00_core.ipynb
# %% ../nbs/01_core.ipynb
def get_sizes(ctx):
"Get the size of each section of the LLM context"
return {o.tag:{p.title:len(p.children[0]) for p in o.children} for o in ctx.children}

# %% ../nbs/00_core.ipynb
# %% ../nbs/01_core.ipynb
@call_parse
def llms_txt2ctx(
fname:str, # File name to read
Expand Down
34 changes: 13 additions & 21 deletions settings.ini
Original file line number Diff line number Diff line change
@@ -1,37 +1,27 @@
[DEFAULT]
# All sections below are required unless otherwise specified.
# See https://github.com/fastai/nbdev/blob/master/settings.ini for examples.

### Python library ###
repo = llms-txt
lib_name = %(repo)s
version = 0.0.1
lib_name = llms-txt
version = 0.0.2
min_python = 3.8
license = apache2
black_formatting = False

### nbdev ###
doc_path = _docs
lib_path = llms_txt
nbs_path = nbs
recursive = True
tst_flags = notest
put_version_in_init = True
cell_number = False

### Docs ###
branch = main
custom_sidebar = False
doc_host = https://%(user)s.github.io
doc_baseurl = /%(repo)s
git_url = https://github.com/%(user)s/%(repo)s
title = %(lib_name)s

### PyPI ###
doc_host = https://AnswerDotAI.github.io
doc_baseurl = /llms-txt
git_url = https://github.com/AnswerDotAI/llms-txt
title = llms-txt
audience = Developers
author = Jeremy Howard
author_email = [email protected]
copyright = 2024 onwards, %(author)s
copyright = 2024 onwards, Jeremy Howard
description = The /llms.txt file, helping language models use your website
keywords = nbdev jupyter notebook python LLMs NLP
language = English
Expand All @@ -40,8 +30,10 @@ user = AnswerDotAI
requirements = fastcore
conda_user = fastai
console_scripts = llms_txt2ctx=llms_txt.core:llms_txt2ctx

### Optional ###
# dev_requirements =
# package_data =
readme_nb = index.ipynb
allowed_metadata_keys =
allowed_cell_metadata_keys =
jupyter_hooks = False
clean_ids = True
clear_all = False

0 comments on commit 7bd2a1a

Please sign in to comment.