Merge branch 'scrapper' into main

.github/workflows/deploy.yml

@@ -0,0 +1,78 @@
+    name: Build and Push Docker
+
+    on:
+    push:
+        branches: ['main', 'scrapper']
+
+    env:
+    REGISTRY: docker.io
+    IMAGE_NAME: ${{ github.repository }}
+    CONTAINER_NAME: django
+
+
+    jobs:
+    build-and-push-image:
+        runs-on: ubuntu-latest
+        permissions:
+        contents: read
+        packages: write
+
+        steps:
+
+        - name: Checkout repository
+            uses: actions/checkout@v3
+
+        - name: Log in to the Container registry
+            uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
+            with:
+            registry: ${{ env.REGISTRY }}
+            username: ${{ github.actor }}
+            password: ${{ secrets.GITHUB_TOKEN }}
+
+        - name: Extract metadata (tags, labels) for Docker
+            id: meta
+            uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
+            with:
+            images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+        - name: Extract metadata (tags, labels) for Docker
+            id: meta2
+            uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
+            with:
+            images: ${{ env.REGISTRY }}/maxconformance/proxy
+
+        - name: Build and push Docker image
+            uses: docker/[email protected]
+            with:
+            context: .
+            push: true
+            target: builder
+            tags: ${{ steps.meta.outputs.tags }}
+            labels: ${{ steps.meta.outputs.labels }}
+
+        - name: Build and push Docker Proxy image
+            uses: docker/[email protected]
+            with:
+            context: .
+            push: true
+            target: deployer
+            tags: ${{ steps.meta2.outputs.tags }}
+            labels: ${{ steps.meta2.outputs.labels }}
+
+        - name: Configure AWS credentials
+            uses: aws-actions/configure-aws-credentials@v1
+            with:
+            aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+            aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS }}
+            aws-region: ap-south-1
+
+        # - name: Deploy Amazon ECS task definition
+        #   uses: aws-actions/amazon-ecs-deploy-task-definition@v1
+        #   with:
+        #     task-definition: .deploy/ecs-task-definition.json
+        #     service: mc-application
+        #     cluster: ProdCluster
+        #     force-new-deployment: true
+        - name: Update ECS Service
+            run: |
+            aws ecs update-service --cluster ProdCluster --service mc-application --desired-count 1 --force-new-deployment

Dockerfile

@@ -0,0 +1,23 @@
+FROM node:20.6.0-buster as frontend
+ADD . /app
+WORKDIR /app
+RUN npm install
+CMD npm start
+
+FROM python:3.11.5-bullseye as parser
+ADD . /app
+WORKDIR /app/parser
+RUN pip install -r requirements.txt
+CMD python parser.py
+
+FROM python:3.11.5-bullseye as scrapper
+ADD . /app
+WORKDIR /app/scrapper
+RUN pip install -r requirements.txt
+CMD python scrapper.py
+
+FROM python:3.11.5-bullseye as analyzer
+ADD . /app
+WORKDIR /app/analyzer
+RUN pip install -r requirements.txt
+CMD python analyzer.py

analyzer/analyzer.py

@@ -0,0 +1,33 @@
+from fastapi import FastAPI, Request, HTTPException
+from fastapi.middleware.cors import CORSMiddleware
+import uvicorn
+app = FastAPI()
+from seoanalyzer import analyze
+
+
+app = FastAPI()
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+
+def analyze_url(url):
+    output = analyze(url, follow_links=False, analyze_headings=True, analyze_extra_tags=True)
+    return output
+
+
+@app.post('/api/v1/analyzer/')
+async def root(request: Request):
+    try: 
+        payload = await request.json()
+        return analyze_url(payload['url'])
+    except Exception as e:
+        raise HTTPException(status_code=500, detail="Error while analyzing")
+
+
+if __name__ == '__main__':
+    uvicorn.run("analyzer:app", host='0.0.0.0', port=8002, reload=True)

analyzer/requirements.txt

@@ -0,0 +1,23 @@
+annotated-types==0.6.0
+anyio==3.7.1
+beautifulsoup4==4.12.2
+certifi==2023.7.22
+charset-normalizer==3.3.1
+click==8.1.7
+fastapi==0.104.0
+h11==0.14.0
+idna==3.4
+Jinja2==3.1.2
+lxml==4.9.3
+MarkupSafe==2.1.3
+pydantic==2.4.2
+pydantic_core==2.10.1
+pyseoanalyzer==4.0.7
+requests==2.31.0
+sitemap==20191121
+sniffio==1.3.0
+soupsieve==2.5
+starlette==0.27.0
+typing_extensions==4.8.0
+urllib3==2.0.7
+uvicorn==0.23.2

docker-compose.yml

@@ -0,0 +1,50 @@
+version: "3.9"
+services:
+  frontend:
+    image: krravindra/algo-frontend:latest
+    ports:
+      - 3000:3000
+    # volumes:
+    #   - ./:/app
+    # working_dir: /app
+    # command: sh -c "npm i && npm start"
+
+  scrapper:
+    image: krravindra/scrapper:latest
+    ports:
+      - 8000:8000
+    # volumes:
+    #   - ./:/app
+    # working_dir: /app/scrapper
+    # command: sh -c "pip3 install -r requirements.txt && python3 main.py"
+
+  parser:
+    image: krravindra/parser:latest
+    ports:
+      - 8001:8001
+    # volumes:
+    #   - ./:/app
+    # working_dir: /app/parser
+    # command: sh -c "pip3 install -r requirements.txt && python3 parser.py"
+
+
+  analyzer:
+    image: krravindra/analyzer:latest
+    ports:
+      - 8002:8002
+
+  redis:
+    image: redis:6.2
+    ports:
+      - 6379:6379
+
+
+  proxy:
+    image: nginx:latest
+    ports:
+    - 80:80
+    - 443:443
+    volumes:
+    - ./app.conf:/etc/nginx/conf.d/app.conf
+    - ./final.crt:/etc/nginx/final.crt
+    - ./server.key:/etc/nginx/server.key