install pgml from apt

ml-cnpg/Dockerfile

@@ -2,8 +2,6 @@ FROM quay.io/tembo/standard-cnpg:15.3.0-1-795fc99
 USER root
 
 ARG PGML_VERSION=2.7.1
-ARG PGVECTOR_VERSION=0.4.4
-ARG PGEMBEDDING_VERSION=0.1.0
 
 WORKDIR /
 

ml-cnpg/requirements-xformers.txt

@@ -1 +1,29 @@
+accelerate
+auto-gptq; sys_platform == 'linux'
+bitsandbytes
+catboost
+ctransformers
+datasets
+deepspeed
+einops
+huggingface-hub
+InstructorEmbedding
+langchain
+lightgbm
+optimum
+orjson
+peft
+pyarrow==11.0.0
+rouge
+sacrebleu
+sacremoses
+sentence-transformers
+tokenizers
+torch
+torchaudio
+torchvision
+transformers
+transformers-stream-generator
+xformers; sys_platform == 'linux'
 xformers==0.0.20
+xgboost

ml-cnpg/requirements.txt

@@ -1,25 +1,123 @@
-accelerate==0.20.3
-auto-gptq==0.2.2
-ctransformers==0.2.8
-datasets==2.13.0
-deepspeed==0.9.4
-huggingface-hub==0.15.1
+accelerate==0.25.0
+aiohttp==3.9.1
+aiosignal==1.3.1
+annotated-types==0.6.0
+anyio==4.2.0
+async-timeout==4.0.3
+attrs==23.1.0
+auto-gptq==0.6.0
+bitsandbytes==0.41.3.post2
+catboost==1.2.2
+certifi==2023.11.17
+charset-normalizer==3.3.2
+click==8.1.7
+colorama==0.4.6
+coloredlogs==15.0.1
+contourpy==1.2.0
+ctransformers==0.2.27
+cycler==0.12.1
+dataclasses-json==0.6.3
+datasets==2.15.0
+deepspeed==0.12.5
+dill==0.3.7
+einops==0.7.0
+exceptiongroup==1.2.0
+filelock==3.13.1
+fonttools==4.47.0
+frozenlist==1.4.1
+fsspec==2023.10.0
+gekko==1.0.6
+graphviz==0.20.1
+greenlet==3.0.2
+hjson==3.1.0
+huggingface-hub==0.19.4
+humanfriendly==10.0
+idna==3.6
 InstructorEmbedding==1.0.1
-lightgbm==3.3.5
-orjson==3.9.1
-pandas==2.0.2
-rich==13.4.2
+Jinja2==3.1.2
+joblib==1.3.2
+jsonpatch==1.33
+jsonpointer==2.4
+kiwisolver==1.4.5
+langchain==0.0.351
+langchain-community==0.0.4
+langchain-core==0.1.1
+langsmith==0.0.72
+lightgbm==4.1.0
+lxml==4.9.3
+MarkupSafe==2.1.3
+marshmallow==3.20.1
+matplotlib==3.8.2
+mpmath==1.3.0
+multidict==6.0.4
+multiprocess==0.70.15
+mypy-extensions==1.0.0
+networkx==3.2.1
+ninja==1.11.1.1
+nltk==3.8.1
+numpy==1.26.2
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==8.9.2.26
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-nccl-cu12==2.18.1
+nvidia-nvjitlink-cu12==12.3.101
+nvidia-nvtx-cu12==12.1.105
+optimum==1.16.1
+orjson==3.9.10
+packaging==23.2
+pandas==2.1.4
+peft==0.7.1
+Pillow==10.1.0
+plotly==5.18.0
+portalocker==2.8.2
+protobuf==4.25.1
+psutil==5.9.7
+py-cpuinfo==9.0.0
+pyarrow==11.0.0
+pyarrow-hotfix==0.6
+pydantic==2.5.2
+pydantic_core==2.14.5
+pynvml==11.5.0
+pyparsing==3.1.1
+python-dateutil==2.8.2
+pytz==2023.3.post1
+PyYAML==6.0.1
+regex==2023.10.3
+requests==2.31.0
 rouge==1.0.1
-sacrebleu==2.3.1
-sacremoses==0.0.53
-scikit-learn==1.2.2
-sentencepiece==0.1.99
+sacrebleu==2.4.0
+sacremoses==0.1.1
+safetensors==0.4.1
+scikit-learn==1.3.2
+scipy==1.11.4
 sentence-transformers==2.2.2
-torch==2.0.1
-torchaudio==2.0.2
-torchvision==0.15.2
-tqdm==4.65.0
-transformers==4.30.2
-xgboost==1.7.5
-langchain==0.0.202
-einops==0.6.1
+sentencepiece==0.1.99
+six==1.16.0
+sniffio==1.3.0
+SQLAlchemy==2.0.23
+sympy==1.12
+tabulate==0.9.0
+tenacity==8.2.3
+threadpoolctl==3.2.0
+tokenizers==0.15.0
+torch==2.1.2
+torchaudio==2.1.2
+torchvision==0.16.2
+tqdm==4.66.1
+transformers==4.36.2
+transformers-stream-generator==0.0.4
+triton==2.1.0
+typing-inspect==0.9.0
+typing_extensions==4.9.0
+tzdata==2023.3
+urllib3==2.1.0
+xformers==0.0.23.post1
+xgboost==2.0.2
+xxhash==3.4.1
+yarl==1.9.4