Skip to content

Commit ecf4a8a

Browse files
loks0nloks0n
authored
Merge pull request #218 from ishandutta0098/feat-implement-sync-with-meilisearch-python
✨ Add Sync with Meilisearch Python Template
2 parents 22b3f2c + 43fbe71 commit ecf4a8a

6 files changed

Lines changed: 457 additions & 0 deletions

File tree

‎python/sync_with_meilisearch/.gitignore‎

Lines changed: 160 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
# Byte-compiled / optimized / DLL files
2+
__pycache__/
3+
*.py[cod]
4+
*$py.class
5+
6+
# C extensions
7+
*.so
8+
9+
# Distribution / packaging
10+
.Python
11+
build/
12+
develop-eggs/
13+
dist/
14+
downloads/
15+
eggs/
16+
.eggs/
17+
lib/
18+
lib64/
19+
parts/
20+
sdist/
21+
var/
22+
wheels/
23+
share/python-wheels/
24+
*.egg-info/
25+
.installed.cfg
26+
*.egg
27+
MANIFEST
28+
29+
# PyInstaller
30+
# Usually these files are written by a python script from a template
31+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
32+
*.manifest
33+
*.spec
34+
35+
# Installer logs
36+
pip-log.txt
37+
pip-delete-this-directory.txt
38+
39+
# Unit test / coverage reports
40+
htmlcov/
41+
.tox/
42+
.nox/
43+
.coverage
44+
.coverage.*
45+
.cache
46+
nosetests.xml
47+
coverage.xml
48+
*.cover
49+
*.py,cover
50+
.hypothesis/
51+
.pytest_cache/
52+
cover/
53+
54+
# Translations
55+
*.mo
56+
*.pot
57+
58+
# Django stuff:
59+
*.log
60+
local_settings.py
61+
db.sqlite3
62+
db.sqlite3-journal
63+
64+
# Flask stuff:
65+
instance/
66+
.webassets-cache
67+
68+
# Scrapy stuff:
69+
.scrapy
70+
71+
# Sphinx documentation
72+
docs/_build/
73+
74+
# PyBuilder
75+
.pybuilder/
76+
target/
77+
78+
# Jupyter Notebook
79+
.ipynb_checkpoints
80+
81+
# IPython
82+
profile_default/
83+
ipython_config.py
84+
85+
# pyenv
86+
# For a library or package, you might want to ignore these files since the code is
87+
# intended to run in multiple environments; otherwise, check them in:
88+
# .python-version
89+
90+
# pipenv
91+
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
92+
# However, in case of collaboration, if having platform-specific dependencies or dependencies
93+
# having no cross-platform support, pipenv may install dependencies that don't work, or not
94+
# install all needed dependencies.
95+
#Pipfile.lock
96+
97+
# poetry
98+
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
99+
# This is especially recommended for binary packages to ensure reproducibility, and is more
100+
# commonly ignored for libraries.
101+
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
102+
#poetry.lock
103+
104+
# pdm
105+
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
106+
#pdm.lock
107+
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
108+
# in version control.
109+
# https://pdm.fming.dev/#use-with-ide
110+
.pdm.toml
111+
112+
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
113+
__pypackages__/
114+
115+
# Celery stuff
116+
celerybeat-schedule
117+
celerybeat.pid
118+
119+
# SageMath parsed files
120+
*.sage.py
121+
122+
# Environments
123+
.env
124+
.venv
125+
env/
126+
venv/
127+
ENV/
128+
env.bak/
129+
venv.bak/
130+
131+
# Spyder project settings
132+
.spyderproject
133+
.spyproject
134+
135+
# Rope project settings
136+
.ropeproject
137+
138+
# mkdocs documentation
139+
/site
140+
141+
# mypy
142+
.mypy_cache/
143+
.dmypy.json
144+
dmypy.json
145+
146+
# Pyre type checker
147+
.pyre/
148+
149+
# pytype static type analyzer
150+
.pytype/
151+
152+
# Cython debug symbols
153+
cython_debug/
154+
155+
# PyCharm
156+
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
157+
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
158+
# and can be added to the global gitignore or merged into this file. For a more nuclear
159+
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
160+
#.idea/

‎python/sync_with_meilisearch/README.md‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
# ⚡ Python Sync with Meilisearch Function
2+
3+
Syncs documents in an Appwrite database collection to a Meilisearch index.
4+
5+
## 🧰 Usage
6+
7+
### GET /
8+
9+
Returns HTML page where search can be performed to test the indexing.
10+
11+
### POST /
12+
13+
Triggers indexing of the Appwrite database collection to Meilisearch.
14+
15+
**Response**
16+
17+
Sample `204` Response: No content.
18+
19+
## ⚙️ Configuration
20+
21+
| Setting | Value |
22+
| ----------------- | --------------------------------- |
23+
| Runtime | Python (3.9) |
24+
| Entrypoint | `src/main.py` |
25+
| Build Commands | `pip install -r requirements.txt` |
26+
| Permissions | `any` |
27+
| Timeout (Seconds) | 15 |
28+
29+
## 🔒 Environment Variables
30+
31+
### APPWRITE_API_KEY
32+
33+
API Key to talk to Appwrite backend APIs.
34+
35+
| Question | Answer |
36+
| ------------- | -------------------------------------------------------------------------------------------------- |
37+
| Required | Yes |
38+
| Sample Value | `d1efb...aec35` |
39+
| Documentation | [Appwrite: Getting Started for Server](https://appwrite.io/docs/getting-started-for-server#apiKey) |
40+
41+
### APPWRITE_DATABASE_ID
42+
43+
The ID of the Appwrite database that contains the collection to sync.
44+
45+
| Question | Answer |
46+
| ------------- | --------------------------------------------------------- |
47+
| Required | Yes |
48+
| Sample Value | `612a3...5b6c9` |
49+
| Documentation | [Appwrite: Databases](https://appwrite.io/docs/databases) |
50+
51+
### APPWRITE_COLLECTION_ID
52+
53+
The ID of the collection in the Appwrite database to sync.
54+
55+
| Question | Answer |
56+
| ------------- | ------------------------------------------------------------- |
57+
| Required | Yes |
58+
| Sample Value | `7c3e8...2a9f1` |
59+
| Documentation | [Appwrite: Collections](https://appwrite.io/docs/databases#collection) |
60+
61+
### APPWRITE_ENDPOINT
62+
63+
The URL endpoint of the Appwrite server. If not provided, it defaults to the Appwrite Cloud server: `https://cloud.appwrite.io/v1`.
64+
65+
| Question | Answer |
66+
| ------------ | ------------------------------ |
67+
| Required | No |
68+
| Sample Value | `https://cloud.appwrite.io/v1` |
69+
70+
### MEILISEARCH_ENDPOINT
71+
72+
The host URL of the Meilisearch server.
73+
74+
| Question | Answer |
75+
| ------------ | ----------------------- |
76+
| Required | Yes |
77+
| Sample Value | `http://127.0.0.1:7700` |
78+
79+
### MEILISEARCH_ADMIN_API_KEY
80+
81+
The admin API key for Meilisearch.
82+
83+
| Question | Answer |
84+
| ------------- | ------------------------------------------------------------------------ |
85+
| Required | Yes |
86+
| Sample Value | `masterKey1234` |
87+
| Documentation | [Meilisearch: API Keys](https://docs.meilisearch.com/reference/api/keys) |
88+
89+
### MEILISEARCH_INDEX_NAME
90+
91+
Name of the Meilisearch index to which the documents will be synchronized.
92+
93+
| Question | Answer |
94+
| ------------ | ---------- |
95+
| Required | Yes |
96+
| Sample Value | `my_index` |
97+
98+
### MEILISEARCH_SEARCH_API_KEY
99+
100+
API Key for Meilisearch search operations.
101+
102+
| Question | Answer |
103+
| ------------- | ------------------------------------------------------------------------ |
104+
| Required | Yes |
105+
| Sample Value | `searchKey1234` |
106+
| Documentation | [Meilisearch: API Keys](https://docs.meilisearch.com/reference/api/keys) |

‎python/sync_with_meilisearch/requirements.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
appwrite
2+
meilisearch

‎python/sync_with_meilisearch/src/main.py‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
import os
2+
from appwrite.client import Client
3+
from appwrite.services.databases import Databases
4+
from meilisearch import Client as MeiliClient
5+
from .utils import get_static_file, interpolate, throw_if_missing
6+
7+
def main(context):
8+
throw_if_missing(os.environ, [
9+
'APPWRITE_API_KEY',
10+
'APPWRITE_DATABASE_ID',
11+
'APPWRITE_COLLECTION_ID',
12+
'MEILISEARCH_ENDPOINT',
13+
'MEILISEARCH_INDEX_NAME',
14+
'MEILISEARCH_ADMIN_API_KEY',
15+
'MEILISEARCH_SEARCH_API_KEY',
16+
])
17+
18+
if context.req.method == 'GET':
19+
html = interpolate(get_static_file('index.html'), {
20+
'MEILISEARCH_ENDPOINT': os.environ['MEILISEARCH_ENDPOINT'],
21+
'MEILISEARCH_INDEX_NAME': os.environ['MEILISEARCH_INDEX_NAME'],
22+
'MEILISEARCH_SEARCH_API_KEY': os.environ['MEILISEARCH_SEARCH_API_KEY'],
23+
})
24+
25+
return context.res.send(html, 200, {'content-type': 'text/html; charset=utf-8'})
26+
27+
client = Client()
28+
client.set_endpoint(os.environ.get('APPWRITE_ENDPOINT', 'https://cloud.appwrite.io/v1'))
29+
client.set_project(os.environ['APPWRITE_FUNCTION_PROJECT_ID'])
30+
client.set_key(os.environ['APPWRITE_API_KEY'])
31+
32+
databases = Databases(client)
33+
34+
meilisearch = MeiliClient(os.environ['MEILISEARCH_ENDPOINT'], os.environ['MEILISEARCH_ADMIN_API_KEY'])
35+
index = meilisearch.index(os.environ['MEILISEARCH_INDEX_NAME'])
36+
37+
cursor = None
38+
39+
while True:
40+
queries = [{'limit': 100}]
41+
42+
if cursor:
43+
queries.append({'cursorAfter': cursor})
44+
45+
response = databases.list_documents(
46+
os.environ['APPWRITE_DATABASE_ID'],
47+
os.environ['APPWRITE_COLLECTION_ID'],
48+
queries
49+
)
50+
51+
documents = response['documents']
52+
53+
if len(documents) > 0:
54+
cursor = documents[-1]['$id']
55+
else:
56+
context.log('No more documents found.')
57+
cursor = None
58+
break
59+
60+
context.log(f'Syncing chunk of {len(documents)} documents...')
61+
index.add_documents(documents, {'primaryKey': '$id'})
62+
63+
context.log('Sync finished.')
64+
65+
return context.res.send('Sync finished.', 200)

‎python/sync_with_meilisearch/src/utils.py‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
import os
2+
import re
3+
4+
__dirname = os.path.dirname(os.path.abspath(__file__))
5+
static_folder = os.path.join(__dirname, "../static")
6+
7+
def throw_if_missing(obj: dict, keys: list[str]) -> None:
8+
"""
9+
Throws an error if any of the keys are missing from the object
10+
11+
Parameters:
12+
obj (dict): Dictionary to check
13+
keys (list[str]): List of keys to check
14+
15+
Raises:
16+
ValueError: If any keys are missing
17+
"""
18+
missing = [key for key in keys if key not in obj or not obj[key]]
19+
if missing:
20+
raise ValueError(f"Missing required fields: {', '.join(missing)}")
21+
22+
def get_static_file(file_name: str) -> str:
23+
"""
24+
Returns the contents of a file in the static folder
25+
26+
Parameters:
27+
file_name (str): Name of the file to read
28+
29+
Returns:
30+
(str): Contents of static/{file_name}
31+
"""
32+
file_path = os.path.join(static_folder, file_name)
33+
with open(file_path, "r") as file:
34+
return file.read()
35+
36+
def interpolate(template: str, values: dict[str, str]) -> str:
37+
"""
38+
Interpolates a template string with the given values
39+
40+
Parameters:
41+
template(str): Template string to interpolate
42+
values(dict): Dictionary of values to interpolate
43+
44+
Returns:
45+
(str): Interpolated string
46+
"""
47+
48+
def replace_match(match):
49+
key = match.group(1)
50+
return values.get(key, "")
51+
52+
return re.sub(r"{{([^}]+)}}", replace_match, template)

0 commit comments

Comments
 (0)