Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Search api #1

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open

Search api #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f9be704
add search api
wtfsayo Nov 6, 2024
26dfcb5
fix meta data fields
wtfsayo Nov 6, 2024
1da11d0
accept empty search
wtfsayo Nov 7, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion daoip-5/scripts/API/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ GitHub repository: [DAOIP-5 JSON files](https://github.com/opensource-observer/o
1. **Clone the repository**:
```bash
git clone https://github.com/opensource-observer/oss-funding
cd oss-funding/daoip-5
cd oss-funding/daoip-5/scripts/API
```

2. **Install dependencies**:
Expand Down
102 changes: 96 additions & 6 deletions daoip-5/scripts/API/run.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
from flask import Flask, jsonify, abort, send_file
import json
import os
import logging

app = Flask(__name__)

# Path to the directory where the JSON files are stored (relative to the repository)
BASE_PATH = '../../json'
BASE_PATH = '../../json'


def get_grant_systems():
Expand Down Expand Up @@ -37,15 +39,15 @@ def display_help():
<body>
<h1>DAOIP-5 Datalake API Documentation</h1>
<p><strong>Repository:</strong> <a href="https://github.com/opensource-observer/oss-funding/tree/main/daoip-5/json" target="_blank">DAOIP-5 JSON Repository</a></p>

<div class="endpoint">
<h2>Endpoint: /</h2>
<p><strong>Method:</strong> GET</p>
<p><strong>Description:</strong> List all grant systems (folders) in the JSON directory.</p>
<p><strong>Parameters:</strong> None</p>
<p><strong>Response:</strong> A JSON array of grant system names.</p>
</div>

<div class="endpoint">
<h2>Endpoint: /&lt;grant_system&gt;</h2>
<p><strong>Method:</strong> GET</p>
Expand All @@ -60,7 +62,7 @@ def display_help():
<p><strong>Response:</strong> A JSON array of JSON file names (grant pools) in the specified folder.</p>
</div>
</div>

<div class="endpoint">
<h2>Endpoint: /&lt;grant_system&gt;/&lt;filename&gt;.json</h2>
<p><strong>Method:</strong> GET</p>
Expand All @@ -76,7 +78,7 @@ def display_help():
<p><strong>Response:</strong> The JSON content of the specified file.</p>
</div>
</div>

<div class="endpoint">
<h2>Endpoint: /help</h2>
<p><strong>Method:</strong> GET</p>
Expand Down Expand Up @@ -151,7 +153,95 @@ def proxy_json_file(grant_system, filename):
except Exception as e:
return jsonify({"error": str(e)}), 500

@app.route('/search/', defaults={'project_name': ''})
@app.route('/search/<project_name>')
def search_project(project_name):
"""
Endpoint to search for all applications matching a project name across all grant systems.
If no project name or empty string is provided, returns all applications.

Args:
project_name (str): The name of the project to search for (optional)
Returns:
JSON object containing matching applications and result count
"""
try:
results = []
# Use existing function to get grant systems
grant_systems = get_grant_systems()

for system in grant_systems:
# Use existing function to get grant pools
files = get_grant_pools(system)
applications_files = [f for f in files if 'applications' in f]

for file in applications_files:
try:
# Use existing function to get file path and Flask's send_file
file_path = get_file_path(system, file)
with open(file_path, 'r') as f:
data = json.load(f)

if not isinstance(data, dict) or 'grant_pools' not in data:
continue

for pool in data['grant_pools']:
if not isinstance(pool, dict) or 'applications' not in pool:
continue

for app in pool['applications']:
# If project_name is empty or None, include all applications
if not project_name:
result = {
**app,
'metadata': {
'grantSystem': system,
'sourceFile': file,
'grantPoolId': app.get('grantPoolId', 'unknown'),
'grantPoolName': app.get('grantPoolName', 'unknown')
}
}
results.append(result)
else:
# More robust project name matching for specific search
project_name_match = str(app.get('projectName', '')).lower()
project_id_match = str(app.get('projectId', '')).lower()
search_term = project_name.lower()

if search_term in project_name_match or search_term in project_id_match:
result = {
**app,
'metadata': {
'grantSystem': system,
'sourceFile': file,
'grantPoolId': app.get('grantPoolId', 'unknown'),
'grantPoolName': app.get('grantPoolName', 'unknown')
}
}
results.append(result)

except (json.JSONDecodeError, Exception) as e:
# Log the error but continue processing other files
logging.error(f"Error processing {file} in {system}: {str(e)}")
continue

search_description = "all applications" if not project_name else f"applications for project: {project_name}"
response = {
"message": f"Found {len(results)} {search_description}",
"count": len(results),
"results": results
}
return jsonify(response), 200

except Exception as e:
error_description = "all applications" if not project_name else f"project: {project_name}"
logging.error(f"Search failed for {error_description}: {str(e)}")
return jsonify({
"error": f"Search failed for {error_description}: {str(e)}",
"status": "error"
}), 500


if __name__ == '__main__':
port = int(os.environ.get("PORT", 5000))
port = int(os.environ.get("PORT", 3000))
app.run(debug=True, host='0.0.0.0', port=port)