From 753806fd6c571bc118b824a853e476f2d10b50db Mon Sep 17 00:00:00 2001 From: vbobrov Date: Wed, 20 Nov 2024 15:52:14 +0700 Subject: [PATCH 1/6] Add mongo-sharding --- mongo-sharding/README.md | 56 +++++++ mongo-sharding/api_app/Dockerfile | 10 ++ mongo-sharding/api_app/app.py | 192 ++++++++++++++++++++++++ mongo-sharding/api_app/requirements.txt | 6 + mongo-sharding/compose.yaml | 133 ++++++++++++++++ mongo-sharding/scripts/mongo-init.sh | 69 +++++++++ 6 files changed, 466 insertions(+) create mode 100644 mongo-sharding/README.md create mode 100644 mongo-sharding/api_app/Dockerfile create mode 100644 mongo-sharding/api_app/app.py create mode 100644 mongo-sharding/api_app/requirements.txt create mode 100644 mongo-sharding/compose.yaml create mode 100644 mongo-sharding/scripts/mongo-init.sh diff --git a/mongo-sharding/README.md b/mongo-sharding/README.md new file mode 100644 index 00000000..081b10ef --- /dev/null +++ b/mongo-sharding/README.md @@ -0,0 +1,56 @@ +# pymongo-api + +Для реализации шардированного варианта приложения реализована упрощенная схема с хэшированной стратегией сегментирования. +Схема включает: 1 роутер, 1 сервер конфигурации и 2 шарда, ссылка на draw.io: + +https://drive.google.com/file/d/1Yd0LxC7VE9Yqa9HL5vOvSke4Psv8tViy/view?usp=sharing + + +## Как запустить + +Запускаем mongodb и приложение + +```shell +docker compose up -d +``` + +Заполняем mongodb данными + +```shell +./scripts/mongo-init.sh +``` + +## Как проверить + +### Рузультат выполнения в shell + +После запуска скрипта консоль выведет общее количество документов (1000) и количесво в каждом шарде. + +```shell +[direct: mongos] somedb> +[direct: mongos] somedb> 1000 +[direct: mongos] somedb> shard1 [direct: primary] test> switched to db somedb +shard1 [direct: primary] somedb> +shard1 [direct: primary] somedb> 492 +shard1 [direct: primary] somedb> shard2 [direct: primary] test> switched to db somedb +shard2 [direct: primary] somedb> +shard2 [direct: primary] somedb> 508 +``` + +### Если вы запускаете проект на локальной машине + +Открыть в браузере http://localhost:8080 + +### Если вы запускаете проект на предоставленной виртуальной машине + +Узнать белый ip виртуальной машины + +```shell +curl --silent http://ifconfig.me +``` + +Откройте в браузере http://:8080 + +## Доступные эндпоинты + +Список доступных эндпоинтов, swagger http://:8080/docs \ No newline at end of file diff --git a/mongo-sharding/api_app/Dockerfile b/mongo-sharding/api_app/Dockerfile new file mode 100644 index 00000000..46f6c9d0 --- /dev/null +++ b/mongo-sharding/api_app/Dockerfile @@ -0,0 +1,10 @@ +FROM python:3.12.1-slim +WORKDIR /app +EXPOSE 8080 +COPY requirements.txt ./ +# Устанавливаем зависимости python не пересобирая их +RUN pip install --no-cache --no-cache-dir -r requirements.txt +# Копирование кода приложения +COPY app.py /app/ +ENTRYPOINT ["uvicorn"] +CMD ["app:app", "--host", "0.0.0.0", "--port", "8080"] diff --git a/mongo-sharding/api_app/app.py b/mongo-sharding/api_app/app.py new file mode 100644 index 00000000..9b19c017 --- /dev/null +++ b/mongo-sharding/api_app/app.py @@ -0,0 +1,192 @@ +import json +import logging +import os +import time +from typing import List, Optional + +import motor.motor_asyncio +from bson import ObjectId +from fastapi import Body, FastAPI, HTTPException, status +from fastapi_cache import FastAPICache +from fastapi_cache.backends.redis import RedisBackend +from fastapi_cache.decorator import cache +from logmiddleware import RouterLoggingMiddleware, logging_config +from pydantic import BaseModel, ConfigDict, EmailStr, Field +from pydantic.functional_validators import BeforeValidator +from pymongo import errors +from redis import asyncio as aioredis +from typing_extensions import Annotated + +# Configure JSON logging +logging.config.dictConfig(logging_config) +logger = logging.getLogger(__name__) + +app = FastAPI() +app.add_middleware( + RouterLoggingMiddleware, + logger=logger, +) + +DATABASE_URL = os.environ["MONGODB_URL"] +DATABASE_NAME = os.environ["MONGODB_DATABASE_NAME"] +REDIS_URL = os.getenv("REDIS_URL", None) + + +def nocache(*args, **kwargs): + def decorator(func): + return func + + return decorator + + +if REDIS_URL: + cache = cache +else: + cache = nocache + + +client = motor.motor_asyncio.AsyncIOMotorClient(DATABASE_URL) +db = client[DATABASE_NAME] + +# Represents an ObjectId field in the database. +# It will be represented as a `str` on the model so that it can be serialized to JSON. +PyObjectId = Annotated[str, BeforeValidator(str)] + + +@app.on_event("startup") +async def startup(): + if REDIS_URL: + redis = aioredis.from_url(REDIS_URL, encoding="utf8", decode_responses=True) + FastAPICache.init(RedisBackend(redis), prefix="api:cache") + + +class UserModel(BaseModel): + """ + Container for a single user record. + """ + + id: Optional[PyObjectId] = Field(alias="_id", default=None) + age: int = Field(...) + name: str = Field(...) + + +class UserCollection(BaseModel): + """ + A container holding a list of `UserModel` instances. + """ + + users: List[UserModel] + + +@app.get("/") +async def root(): + collection_names = await db.list_collection_names() + collections = {} + for collection_name in collection_names: + collection = db.get_collection(collection_name) + collections[collection_name] = { + "documents_count": await collection.count_documents({}) + } + try: + replica_status = await client.admin.command("replSetGetStatus") + replica_status = json.dumps(replica_status, indent=2, default=str) + except errors.OperationFailure: + replica_status = "No Replicas" + + topology_description = client.topology_description + read_preference = client.client_options.read_preference + topology_type = topology_description.topology_type_name + replicaset_name = topology_description.replica_set_name + + shards = None + if topology_type == "Sharded": + shards_list = await client.admin.command("listShards") + shards = {} + for shard in shards_list.get("shards", {}): + shards[shard["_id"]] = shard["host"] + + cache_enabled = False + if REDIS_URL: + cache_enabled = FastAPICache.get_enable() + + return { + "mongo_topology_type": topology_type, + "mongo_replicaset_name": replicaset_name, + "mongo_db": DATABASE_NAME, + "read_preference": str(read_preference), + "mongo_nodes": client.nodes, + "mongo_primary_host": client.primary, + "mongo_secondary_hosts": client.secondaries, + "mongo_address": client.address, + "mongo_is_primary": client.is_primary, + "mongo_is_mongos": client.is_mongos, + "collections": collections, + "shards": shards, + "cache_enabled": cache_enabled, + "status": "OK", + } + + +@app.get("/{collection_name}/count") +async def collection_count(collection_name: str): + collection = db.get_collection(collection_name) + items_count = await collection.count_documents({}) + # status = await client.admin.command('replSetGetStatus') + # import ipdb; ipdb.set_trace() + return {"status": "OK", "mongo_db": DATABASE_NAME, "items_count": items_count} + + +@app.get( + "/{collection_name}/users", + response_description="List all users", + response_model=UserCollection, + response_model_by_alias=False, +) +@cache(expire=60 * 1) +async def list_users(collection_name: str): + """ + List all of the user data in the database. + The response is unpaginated and limited to 1000 results. + """ + time.sleep(1) + collection = db.get_collection(collection_name) + return UserCollection(users=await collection.find().to_list(1000)) + + +@app.get( + "/{collection_name}/users/{name}", + response_description="Get a single user", + response_model=UserModel, + response_model_by_alias=False, +) +async def show_user(collection_name: str, name: str): + """ + Get the record for a specific user, looked up by `name`. + """ + + collection = db.get_collection(collection_name) + if (user := await collection.find_one({"name": name})) is not None: + return user + + raise HTTPException(status_code=404, detail=f"User {name} not found") + + +@app.post( + "/{collection_name}/users", + response_description="Add new user", + response_model=UserModel, + status_code=status.HTTP_201_CREATED, + response_model_by_alias=False, +) +async def create_user(collection_name: str, user: UserModel = Body(...)): + """ + Insert a new user record. + + A unique `id` will be created and provided in the response. + """ + collection = db.get_collection(collection_name) + new_user = await collection.insert_one( + user.model_dump(by_alias=True, exclude=["id"]) + ) + created_user = await collection.find_one({"_id": new_user.inserted_id}) + return created_user diff --git a/mongo-sharding/api_app/requirements.txt b/mongo-sharding/api_app/requirements.txt new file mode 100644 index 00000000..61b29b87 --- /dev/null +++ b/mongo-sharding/api_app/requirements.txt @@ -0,0 +1,6 @@ +fastapi==0.110.2 +uvicorn[standard]==0.29.0 +motor==3.5.3 +redis==4.4.2 +fastapi-cache2==0.2.0 +logmiddleware==0.0.4 \ No newline at end of file diff --git a/mongo-sharding/compose.yaml b/mongo-sharding/compose.yaml new file mode 100644 index 00000000..485e6c87 --- /dev/null +++ b/mongo-sharding/compose.yaml @@ -0,0 +1,133 @@ +services: + + configSrv: + image: mongo:latest + container_name: configSrv + restart: always + ports: + - "27017:27017" + networks: + app-network: + ipv4_address: 173.17.0.10 + volumes: + - config-data:/data/db + command: + [ + "--configsvr", + "--replSet", + "config_server", + "--bind_ip_all", + "--port", + "27017" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + + shard1: + image: mongo:latest + container_name: shard1 + restart: always + ports: + - "27018:27018" + networks: + app-network: + ipv4_address: 173.17.0.9 + volumes: + - shard1-data:/data/db + command: + [ + "--shardsvr", + "--replSet", + "shard1", + "--bind_ip_all", + "--port", + "27018" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + + + shard2: + image: mongo:latest + container_name: shard2 + restart: always + ports: + - "27019:27019" + networks: + app-network: + ipv4_address: 173.17.0.8 + volumes: + - shard2-data:/data/db + command: + [ + "--shardsvr", + "--replSet", + "shard2", + "--bind_ip_all", + "--port", + "27019" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + + mongos_router: + image: mongo:latest + container_name: mongos_router + restart: always + ports: + - "27020:27020" + networks: + app-network: + ipv4_address: 173.17.0.7 + command: + [ + "mongos", + "--configdb", + "config_server/configSrv:27017", + "--bind_ip_all", + "--port", + "27020" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + + + pymongo_api: + container_name: pymongo_api + build: + context: api_app + dockerfile: Dockerfile + image: kazhem/pymongo_api:1.0.0 + depends_on: + - mongos_router + ports: + - 8080:8080 + networks: + app-network: + ipv4_address: 173.17.0.11 + environment: + MONGODB_URL: "mongodb://mongos_router:27020" + MONGODB_DATABASE_NAME: "somedb" + + +networks: + app-network: + driver: bridge + ipam: + driver: default + config: + - subnet: 173.17.0.0/16 + + +volumes: + config-data: + shard1-data: + shard2-data: diff --git a/mongo-sharding/scripts/mongo-init.sh b/mongo-sharding/scripts/mongo-init.sh new file mode 100644 index 00000000..95bd9800 --- /dev/null +++ b/mongo-sharding/scripts/mongo-init.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +### +# Инициализируем бд +### + +docker compose exec -T configSrv mongosh --port 27017 --quiet < Date: Thu, 21 Nov 2024 17:31:01 +0700 Subject: [PATCH 2/6] Add mongo-sharding-repl --- mongo-sharding-repl/README.md | 56 ++++++ mongo-sharding-repl/api_app/Dockerfile | 10 + mongo-sharding-repl/api_app/app.py | 192 +++++++++++++++++++ mongo-sharding-repl/api_app/requirements.txt | 6 + mongo-sharding-repl/compose.yaml | 133 +++++++++++++ mongo-sharding-repl/scripts/mongo-init.sh | 69 +++++++ 6 files changed, 466 insertions(+) create mode 100644 mongo-sharding-repl/README.md create mode 100644 mongo-sharding-repl/api_app/Dockerfile create mode 100644 mongo-sharding-repl/api_app/app.py create mode 100644 mongo-sharding-repl/api_app/requirements.txt create mode 100644 mongo-sharding-repl/compose.yaml create mode 100644 mongo-sharding-repl/scripts/mongo-init.sh diff --git a/mongo-sharding-repl/README.md b/mongo-sharding-repl/README.md new file mode 100644 index 00000000..081b10ef --- /dev/null +++ b/mongo-sharding-repl/README.md @@ -0,0 +1,56 @@ +# pymongo-api + +Для реализации шардированного варианта приложения реализована упрощенная схема с хэшированной стратегией сегментирования. +Схема включает: 1 роутер, 1 сервер конфигурации и 2 шарда, ссылка на draw.io: + +https://drive.google.com/file/d/1Yd0LxC7VE9Yqa9HL5vOvSke4Psv8tViy/view?usp=sharing + + +## Как запустить + +Запускаем mongodb и приложение + +```shell +docker compose up -d +``` + +Заполняем mongodb данными + +```shell +./scripts/mongo-init.sh +``` + +## Как проверить + +### Рузультат выполнения в shell + +После запуска скрипта консоль выведет общее количество документов (1000) и количесво в каждом шарде. + +```shell +[direct: mongos] somedb> +[direct: mongos] somedb> 1000 +[direct: mongos] somedb> shard1 [direct: primary] test> switched to db somedb +shard1 [direct: primary] somedb> +shard1 [direct: primary] somedb> 492 +shard1 [direct: primary] somedb> shard2 [direct: primary] test> switched to db somedb +shard2 [direct: primary] somedb> +shard2 [direct: primary] somedb> 508 +``` + +### Если вы запускаете проект на локальной машине + +Открыть в браузере http://localhost:8080 + +### Если вы запускаете проект на предоставленной виртуальной машине + +Узнать белый ip виртуальной машины + +```shell +curl --silent http://ifconfig.me +``` + +Откройте в браузере http://:8080 + +## Доступные эндпоинты + +Список доступных эндпоинтов, swagger http://:8080/docs \ No newline at end of file diff --git a/mongo-sharding-repl/api_app/Dockerfile b/mongo-sharding-repl/api_app/Dockerfile new file mode 100644 index 00000000..46f6c9d0 --- /dev/null +++ b/mongo-sharding-repl/api_app/Dockerfile @@ -0,0 +1,10 @@ +FROM python:3.12.1-slim +WORKDIR /app +EXPOSE 8080 +COPY requirements.txt ./ +# Устанавливаем зависимости python не пересобирая их +RUN pip install --no-cache --no-cache-dir -r requirements.txt +# Копирование кода приложения +COPY app.py /app/ +ENTRYPOINT ["uvicorn"] +CMD ["app:app", "--host", "0.0.0.0", "--port", "8080"] diff --git a/mongo-sharding-repl/api_app/app.py b/mongo-sharding-repl/api_app/app.py new file mode 100644 index 00000000..9b19c017 --- /dev/null +++ b/mongo-sharding-repl/api_app/app.py @@ -0,0 +1,192 @@ +import json +import logging +import os +import time +from typing import List, Optional + +import motor.motor_asyncio +from bson import ObjectId +from fastapi import Body, FastAPI, HTTPException, status +from fastapi_cache import FastAPICache +from fastapi_cache.backends.redis import RedisBackend +from fastapi_cache.decorator import cache +from logmiddleware import RouterLoggingMiddleware, logging_config +from pydantic import BaseModel, ConfigDict, EmailStr, Field +from pydantic.functional_validators import BeforeValidator +from pymongo import errors +from redis import asyncio as aioredis +from typing_extensions import Annotated + +# Configure JSON logging +logging.config.dictConfig(logging_config) +logger = logging.getLogger(__name__) + +app = FastAPI() +app.add_middleware( + RouterLoggingMiddleware, + logger=logger, +) + +DATABASE_URL = os.environ["MONGODB_URL"] +DATABASE_NAME = os.environ["MONGODB_DATABASE_NAME"] +REDIS_URL = os.getenv("REDIS_URL", None) + + +def nocache(*args, **kwargs): + def decorator(func): + return func + + return decorator + + +if REDIS_URL: + cache = cache +else: + cache = nocache + + +client = motor.motor_asyncio.AsyncIOMotorClient(DATABASE_URL) +db = client[DATABASE_NAME] + +# Represents an ObjectId field in the database. +# It will be represented as a `str` on the model so that it can be serialized to JSON. +PyObjectId = Annotated[str, BeforeValidator(str)] + + +@app.on_event("startup") +async def startup(): + if REDIS_URL: + redis = aioredis.from_url(REDIS_URL, encoding="utf8", decode_responses=True) + FastAPICache.init(RedisBackend(redis), prefix="api:cache") + + +class UserModel(BaseModel): + """ + Container for a single user record. + """ + + id: Optional[PyObjectId] = Field(alias="_id", default=None) + age: int = Field(...) + name: str = Field(...) + + +class UserCollection(BaseModel): + """ + A container holding a list of `UserModel` instances. + """ + + users: List[UserModel] + + +@app.get("/") +async def root(): + collection_names = await db.list_collection_names() + collections = {} + for collection_name in collection_names: + collection = db.get_collection(collection_name) + collections[collection_name] = { + "documents_count": await collection.count_documents({}) + } + try: + replica_status = await client.admin.command("replSetGetStatus") + replica_status = json.dumps(replica_status, indent=2, default=str) + except errors.OperationFailure: + replica_status = "No Replicas" + + topology_description = client.topology_description + read_preference = client.client_options.read_preference + topology_type = topology_description.topology_type_name + replicaset_name = topology_description.replica_set_name + + shards = None + if topology_type == "Sharded": + shards_list = await client.admin.command("listShards") + shards = {} + for shard in shards_list.get("shards", {}): + shards[shard["_id"]] = shard["host"] + + cache_enabled = False + if REDIS_URL: + cache_enabled = FastAPICache.get_enable() + + return { + "mongo_topology_type": topology_type, + "mongo_replicaset_name": replicaset_name, + "mongo_db": DATABASE_NAME, + "read_preference": str(read_preference), + "mongo_nodes": client.nodes, + "mongo_primary_host": client.primary, + "mongo_secondary_hosts": client.secondaries, + "mongo_address": client.address, + "mongo_is_primary": client.is_primary, + "mongo_is_mongos": client.is_mongos, + "collections": collections, + "shards": shards, + "cache_enabled": cache_enabled, + "status": "OK", + } + + +@app.get("/{collection_name}/count") +async def collection_count(collection_name: str): + collection = db.get_collection(collection_name) + items_count = await collection.count_documents({}) + # status = await client.admin.command('replSetGetStatus') + # import ipdb; ipdb.set_trace() + return {"status": "OK", "mongo_db": DATABASE_NAME, "items_count": items_count} + + +@app.get( + "/{collection_name}/users", + response_description="List all users", + response_model=UserCollection, + response_model_by_alias=False, +) +@cache(expire=60 * 1) +async def list_users(collection_name: str): + """ + List all of the user data in the database. + The response is unpaginated and limited to 1000 results. + """ + time.sleep(1) + collection = db.get_collection(collection_name) + return UserCollection(users=await collection.find().to_list(1000)) + + +@app.get( + "/{collection_name}/users/{name}", + response_description="Get a single user", + response_model=UserModel, + response_model_by_alias=False, +) +async def show_user(collection_name: str, name: str): + """ + Get the record for a specific user, looked up by `name`. + """ + + collection = db.get_collection(collection_name) + if (user := await collection.find_one({"name": name})) is not None: + return user + + raise HTTPException(status_code=404, detail=f"User {name} not found") + + +@app.post( + "/{collection_name}/users", + response_description="Add new user", + response_model=UserModel, + status_code=status.HTTP_201_CREATED, + response_model_by_alias=False, +) +async def create_user(collection_name: str, user: UserModel = Body(...)): + """ + Insert a new user record. + + A unique `id` will be created and provided in the response. + """ + collection = db.get_collection(collection_name) + new_user = await collection.insert_one( + user.model_dump(by_alias=True, exclude=["id"]) + ) + created_user = await collection.find_one({"_id": new_user.inserted_id}) + return created_user diff --git a/mongo-sharding-repl/api_app/requirements.txt b/mongo-sharding-repl/api_app/requirements.txt new file mode 100644 index 00000000..61b29b87 --- /dev/null +++ b/mongo-sharding-repl/api_app/requirements.txt @@ -0,0 +1,6 @@ +fastapi==0.110.2 +uvicorn[standard]==0.29.0 +motor==3.5.3 +redis==4.4.2 +fastapi-cache2==0.2.0 +logmiddleware==0.0.4 \ No newline at end of file diff --git a/mongo-sharding-repl/compose.yaml b/mongo-sharding-repl/compose.yaml new file mode 100644 index 00000000..485e6c87 --- /dev/null +++ b/mongo-sharding-repl/compose.yaml @@ -0,0 +1,133 @@ +services: + + configSrv: + image: mongo:latest + container_name: configSrv + restart: always + ports: + - "27017:27017" + networks: + app-network: + ipv4_address: 173.17.0.10 + volumes: + - config-data:/data/db + command: + [ + "--configsvr", + "--replSet", + "config_server", + "--bind_ip_all", + "--port", + "27017" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + + shard1: + image: mongo:latest + container_name: shard1 + restart: always + ports: + - "27018:27018" + networks: + app-network: + ipv4_address: 173.17.0.9 + volumes: + - shard1-data:/data/db + command: + [ + "--shardsvr", + "--replSet", + "shard1", + "--bind_ip_all", + "--port", + "27018" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + + + shard2: + image: mongo:latest + container_name: shard2 + restart: always + ports: + - "27019:27019" + networks: + app-network: + ipv4_address: 173.17.0.8 + volumes: + - shard2-data:/data/db + command: + [ + "--shardsvr", + "--replSet", + "shard2", + "--bind_ip_all", + "--port", + "27019" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + + mongos_router: + image: mongo:latest + container_name: mongos_router + restart: always + ports: + - "27020:27020" + networks: + app-network: + ipv4_address: 173.17.0.7 + command: + [ + "mongos", + "--configdb", + "config_server/configSrv:27017", + "--bind_ip_all", + "--port", + "27020" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + + + pymongo_api: + container_name: pymongo_api + build: + context: api_app + dockerfile: Dockerfile + image: kazhem/pymongo_api:1.0.0 + depends_on: + - mongos_router + ports: + - 8080:8080 + networks: + app-network: + ipv4_address: 173.17.0.11 + environment: + MONGODB_URL: "mongodb://mongos_router:27020" + MONGODB_DATABASE_NAME: "somedb" + + +networks: + app-network: + driver: bridge + ipam: + driver: default + config: + - subnet: 173.17.0.0/16 + + +volumes: + config-data: + shard1-data: + shard2-data: diff --git a/mongo-sharding-repl/scripts/mongo-init.sh b/mongo-sharding-repl/scripts/mongo-init.sh new file mode 100644 index 00000000..95bd9800 --- /dev/null +++ b/mongo-sharding-repl/scripts/mongo-init.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +### +# Инициализируем бд +### + +docker compose exec -T configSrv mongosh --port 27017 --quiet < Date: Thu, 21 Nov 2024 18:06:49 +0700 Subject: [PATCH 3/6] Add peplication --- mongo-sharding-repl/compose.yaml | 61 +++++++++++++++++++++-- mongo-sharding-repl/scripts/mongo-init.sh | 20 ++++++-- 2 files changed, 74 insertions(+), 7 deletions(-) diff --git a/mongo-sharding-repl/compose.yaml b/mongo-sharding-repl/compose.yaml index 485e6c87..c121d83a 100644 --- a/mongo-sharding-repl/compose.yaml +++ b/mongo-sharding-repl/compose.yaml @@ -25,9 +25,9 @@ services: interval: 5s start_period: 10s - shard1: + shard1_db1: image: mongo:latest - container_name: shard1 + container_name: shard1_db1 restart: always ports: - "27018:27018" @@ -35,7 +35,7 @@ services: app-network: ipv4_address: 173.17.0.9 volumes: - - shard1-data:/data/db + - shard1_db1-data:/data/db command: [ "--shardsvr", @@ -50,6 +50,57 @@ services: interval: 5s start_period: 10s + shard1_db2: + image: mongo:latest + container_name: shard1_db2 + restart: always + ports: + - "27028:27028" + networks: + app-network: + ipv4_address: 173.17.0.12 + volumes: + - shard1_db2-data:/data/db + command: + [ + "--shardsvr", + "--replSet", + "shard1", + "--bind_ip_all", + "--port", + "27028" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + + + shard1_db3: + image: mongo:latest + container_name: shard1_db3 + restart: always + ports: + - "27038:27038" + networks: + app-network: + ipv4_address: 173.17.0.13 + volumes: + - shard1_db3-data:/data/db + command: + [ + "--shardsvr", + "--replSet", + "shard1", + "--bind_ip_all", + "--port", + "27038" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + shard2: image: mongo:latest @@ -129,5 +180,7 @@ networks: volumes: config-data: - shard1-data: + shard1_db1-data: + shard1_db2-data: + shard1_db3-data: shard2-data: diff --git a/mongo-sharding-repl/scripts/mongo-init.sh b/mongo-sharding-repl/scripts/mongo-init.sh index 95bd9800..36c28506 100644 --- a/mongo-sharding-repl/scripts/mongo-init.sh +++ b/mongo-sharding-repl/scripts/mongo-init.sh @@ -16,12 +16,14 @@ rs.initiate( ); EOF -docker compose exec -T shard1 mongosh --port 27018 --quiet < Date: Fri, 22 Nov 2024 18:13:29 +0700 Subject: [PATCH 4/6] Add shard2 replication --- mongo-sharding-repl/README.md | 47 +++++++++---- mongo-sharding-repl/compose.yaml | 75 +++++++++++++++++++-- mongo-sharding-repl/scripts/db-drop.sh | 11 +++ mongo-sharding-repl/scripts/mongo-init.sh | 45 ++----------- mongo-sharding-repl/scripts/print-output.sh | 52 ++++++++++++++ mongo-sharding/README.md | 6 ++ mongo-sharding/compose.yaml | 1 + mongo-sharding/scripts/db-drop.sh | 11 +++ mongo-sharding/scripts/mongo-init.sh | 17 ++--- 9 files changed, 197 insertions(+), 68 deletions(-) create mode 100644 mongo-sharding-repl/scripts/db-drop.sh create mode 100644 mongo-sharding-repl/scripts/print-output.sh create mode 100644 mongo-sharding/scripts/db-drop.sh diff --git a/mongo-sharding-repl/README.md b/mongo-sharding-repl/README.md index 081b10ef..5d6154ab 100644 --- a/mongo-sharding-repl/README.md +++ b/mongo-sharding-repl/README.md @@ -1,9 +1,9 @@ # pymongo-api -Для реализации шардированного варианта приложения реализована упрощенная схема с хэшированной стратегией сегментирования. -Схема включает: 1 роутер, 1 сервер конфигурации и 2 шарда, ссылка на draw.io: +Для реализации шардированного варианта с репликацией приложения реализована упрощенная схема с хэшированной стратегией сегментирования и тремя репликами на каждом шарде. +Схема включает: 1 роутер, 1 сервер конфигурации и 2 шарда с тремя репликами в каждом, ссылка на draw.io: -https://drive.google.com/file/d/1Yd0LxC7VE9Yqa9HL5vOvSke4Psv8tViy/view?usp=sharing +https://drive.google.com/file/d/1DtlZUV0B0FKdAe5qVzqNcSVcwjM4ETIT/view?usp=sharing ## Как запустить @@ -14,27 +14,48 @@ https://drive.google.com/file/d/1Yd0LxC7VE9Yqa9HL5vOvSke4Psv8tViy/view?usp=shari docker compose up -d ``` -Заполняем mongodb данными +Проводим инициализацию системы ```shell ./scripts/mongo-init.sh ``` +Наполняеи mongodb данными и выводим результаты в консоль + +```shell +./scripts/print-output.sh +``` + +Очистка базы данных + +```shell +./scripts/db-drop.sh +``` + + ## Как проверить ### Рузультат выполнения в shell -После запуска скрипта консоль выведет общее количество документов (1000) и количесво в каждом шарде. +После запуска скриптов консоль выведет общее количество документов (+1000), количесво реплик (3) и количество в каждом шарде. ```shell -[direct: mongos] somedb> -[direct: mongos] somedb> 1000 -[direct: mongos] somedb> shard1 [direct: primary] test> switched to db somedb -shard1 [direct: primary] somedb> -shard1 [direct: primary] somedb> 492 -shard1 [direct: primary] somedb> shard2 [direct: primary] test> switched to db somedb -shard2 [direct: primary] somedb> -shard2 [direct: primary] somedb> 508 +[direct: mongos] somedb> NOTE: helloDoc count=1000 +[direct: mongos] somedb> shard1 [direct: primary] test> OUTPUT: shard1 repl count=3 +shard1 [direct: primary] test> shard1 [direct: primary] test> switched to db somedb +shard1 [direct: primary] somedb> NOTE: shard1_db1 doc count=492 +shard1 [direct: primary] somedb> shard1 [direct: secondary] test> switched to db somedb +shard1 [direct: secondary] somedb> NOTE: shard1_db2 doc count=492 +shard1 [direct: secondary] somedb> shard1 [direct: secondary] test> switched to db somedb +shard1 [direct: secondary] somedb> NOTE: shard1_db3 doc count=492 +shard1 [direct: secondary] somedb> shard2 [direct: primary] test> +shard2 [direct: primary] test> OUTPUT: shard2 repl count=3 +shard2 [direct: primary] test> shard2 [direct: primary] test> switched to db somedb +shard2 [direct: primary] somedb> NOTE: shard2_db1 doc count=508 +shard2 [direct: primary] somedb> shard2 [direct: secondary] test> switched to db somedb +shard2 [direct: secondary] somedb> NOTE: shard2_db2 doc count=508 +shard2 [direct: secondary] somedb> shard2 [direct: secondary] test> switched to db somedb +shard2 [direct: secondary] somedb> NOTE: shard2_db3 doc count=508 ``` ### Если вы запускаете проект на локальной машине diff --git a/mongo-sharding-repl/compose.yaml b/mongo-sharding-repl/compose.yaml index c121d83a..f62d524e 100644 --- a/mongo-sharding-repl/compose.yaml +++ b/mongo-sharding-repl/compose.yaml @@ -1,3 +1,4 @@ +name: mongо-sharding-repl services: configSrv: @@ -102,9 +103,9 @@ services: start_period: 10s - shard2: + shard2_db1: image: mongo:latest - container_name: shard2 + container_name: shard2_db1 restart: always ports: - "27019:27019" @@ -112,7 +113,7 @@ services: app-network: ipv4_address: 173.17.0.8 volumes: - - shard2-data:/data/db + - shard2_db1-data:/data/db command: [ "--shardsvr", @@ -126,6 +127,56 @@ services: test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] interval: 5s start_period: 10s + + shard2_db2: + image: mongo:latest + container_name: shard2_db2 + restart: always + ports: + - "27029:27029" + networks: + app-network: + ipv4_address: 173.17.0.15 + volumes: + - shard2_db2-data:/data/db + command: + [ + "--shardsvr", + "--replSet", + "shard2", + "--bind_ip_all", + "--port", + "27029" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + + shard2_db3: + image: mongo:latest + container_name: shard2_db3 + restart: always + ports: + - "27039:27039" + networks: + app-network: + ipv4_address: 173.17.0.16 + volumes: + - shard2_db3-data:/data/db + command: + [ + "--shardsvr", + "--replSet", + "shard2", + "--bind_ip_all", + "--port", + "27039" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s mongos_router: image: mongo:latest @@ -151,6 +202,18 @@ services: start_period: 10s + redis_1: + image: "redis:latest" + container_name: redis_1 + ports: + - "6379" + volumes: + - redis_1_data:/data + networks: + app-network: + ipv4_address: 173.17.0.14 + + pymongo_api: container_name: pymongo_api build: @@ -167,6 +230,7 @@ services: environment: MONGODB_URL: "mongodb://mongos_router:27020" MONGODB_DATABASE_NAME: "somedb" + REDIS_URL: "redis://redis_1:6379" networks: @@ -183,4 +247,7 @@ volumes: shard1_db1-data: shard1_db2-data: shard1_db3-data: - shard2-data: + shard2_db1-data: + shard2_db2-data: + shard2_db3-data: + redis_1_data: {} diff --git a/mongo-sharding-repl/scripts/db-drop.sh b/mongo-sharding-repl/scripts/db-drop.sh new file mode 100644 index 00000000..b4292c4f --- /dev/null +++ b/mongo-sharding-repl/scripts/db-drop.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +### +# Очистка бд +### + +docker compose exec -T mongos_router mongosh --port 27020 --quiet < Date: Sat, 23 Nov 2024 11:53:49 +0700 Subject: [PATCH 5/6] Add sharding-repl-cache --- mongo-sharding-repl/README.md | 2 +- .../{print-output.sh => load-output.sh} | 0 sharding-repl-cache/README.md | 99 +++++++ sharding-repl-cache/api_app/Dockerfile | 10 + sharding-repl-cache/api_app/app.py | 192 +++++++++++++ sharding-repl-cache/api_app/requirements.txt | 6 + sharding-repl-cache/compose.yaml | 253 ++++++++++++++++++ sharding-repl-cache/scripts/db-drop.sh | 11 + sharding-repl-cache/scripts/load-output.sh | 52 ++++ sharding-repl-cache/scripts/mongo-init.sh | 50 ++++ sharding-repl-cache/scripts/print.sh | 9 + 11 files changed, 683 insertions(+), 1 deletion(-) rename mongo-sharding-repl/scripts/{print-output.sh => load-output.sh} (100%) create mode 100644 sharding-repl-cache/README.md create mode 100644 sharding-repl-cache/api_app/Dockerfile create mode 100644 sharding-repl-cache/api_app/app.py create mode 100644 sharding-repl-cache/api_app/requirements.txt create mode 100644 sharding-repl-cache/compose.yaml create mode 100644 sharding-repl-cache/scripts/db-drop.sh create mode 100644 sharding-repl-cache/scripts/load-output.sh create mode 100644 sharding-repl-cache/scripts/mongo-init.sh create mode 100644 sharding-repl-cache/scripts/print.sh diff --git a/mongo-sharding-repl/README.md b/mongo-sharding-repl/README.md index 5d6154ab..127adde0 100644 --- a/mongo-sharding-repl/README.md +++ b/mongo-sharding-repl/README.md @@ -23,7 +23,7 @@ docker compose up -d Наполняеи mongodb данными и выводим результаты в консоль ```shell -./scripts/print-output.sh +./scripts/load-output.sh ``` Очистка базы данных diff --git a/mongo-sharding-repl/scripts/print-output.sh b/mongo-sharding-repl/scripts/load-output.sh similarity index 100% rename from mongo-sharding-repl/scripts/print-output.sh rename to mongo-sharding-repl/scripts/load-output.sh diff --git a/sharding-repl-cache/README.md b/sharding-repl-cache/README.md new file mode 100644 index 00000000..32086f31 --- /dev/null +++ b/sharding-repl-cache/README.md @@ -0,0 +1,99 @@ +# pymongo-api + +Для реализации шардированного варианта с репликацией и кэшированием реализована упрощенная схема с хэшированной стратегией сегментирования и тремя репликами на каждом шарде. Для временного хранения данных реализован сервис хэширования Redis в Single Node режиме. В приложении кэширование доступно для эндпоинта //users +Схема включает: 1 роутер, 1 сервер конфигурации и 2 шарда с тремя репликами в каждом, 1 сервис хэширования, ссылка на draw.io: + +https://drive.google.com/file/d/1h76DW9y2f82uBVmRDcxmS2nfHk3ApoDa/view?usp=sharing + + +## Что реализовано + +В файле соmpose.yaml сфомирована реализация двух шардов mongo-db c тремя репликами в каждой. + +Добавлен Single Node Redis, в инстансе приложения ymongo_api прописана глобальная переменная REDIS_URL: "redis://redis_1:6379" + +Добавлены скрипты запуска bash для инициализации контейнеров и вывода результатов в консоль. + +## Как запустить + +Запускаем mongodb и приложение + +```shell +docker compose up -d +``` + +Проводим инициализацию системы + +```shell +./scripts/mongo-init.sh +``` + +Наполняеи mongodb данными и выводим результаты в консоль + +```shell +./scripts/load-output.sh +``` + +Очистка базы данных + +```shell +./scripts/db-drop.sh +``` + + +## Как проверить + +### Рузультат выполнения в shell + +После запуска скриптов консоль выведет общее количество документов (+1000), количесво реплик (3) и количество в каждом шарде. + +```shell +[direct: mongos] somedb> NOTE: helloDoc count=1000 +[direct: mongos] somedb> shard1 [direct: primary] test> OUTPUT: shard1 repl count=3 +shard1 [direct: primary] test> shard1 [direct: primary] test> switched to db somedb +shard1 [direct: primary] somedb> NOTE: shard1_db1 doc count=492 +shard1 [direct: primary] somedb> shard1 [direct: secondary] test> switched to db somedb +shard1 [direct: secondary] somedb> NOTE: shard1_db2 doc count=492 +shard1 [direct: secondary] somedb> shard1 [direct: secondary] test> switched to db somedb +shard1 [direct: secondary] somedb> NOTE: shard1_db3 doc count=492 +shard1 [direct: secondary] somedb> shard2 [direct: primary] test> +shard2 [direct: primary] test> OUTPUT: shard2 repl count=3 +shard2 [direct: primary] test> shard2 [direct: primary] test> switched to db somedb +shard2 [direct: primary] somedb> NOTE: shard2_db1 doc count=508 +shard2 [direct: primary] somedb> shard2 [direct: secondary] test> switched to db somedb +shard2 [direct: secondary] somedb> NOTE: shard2_db2 doc count=508 +shard2 [direct: secondary] somedb> shard2 [direct: secondary] test> switched to db somedb +shard2 [direct: secondary] somedb> NOTE: shard2_db3 doc count=508 +``` + +Для проверки хэширования необходимо перейти в swagger + +swagger http://:8080/docs + +И в энедпоните POST создания пользователя выполнить запрос + +//users + +Далее в методе GET //users получить пользователя. +Кэширование значительно увеличивает скорость отклика для повторных запросов, второй и последующие вызовы выполяются <100мс. +Сброс кэша происходит автоматически по таймингу. + + +### Если вы запускаете проект на локальной машине + +Открыть в браузере http://localhost:8080 + +### Если вы запускаете проект на предоставленной виртуальной машине + +Узнать белый ip виртуальной машины + +```shell +curl --silent http://ifconfig.me +``` + +Откройте в браузере http://:8080 + +## Доступные эндпоинты + +Список доступных эндпоинтов, swagger http://:8080/docs + diff --git a/sharding-repl-cache/api_app/Dockerfile b/sharding-repl-cache/api_app/Dockerfile new file mode 100644 index 00000000..46f6c9d0 --- /dev/null +++ b/sharding-repl-cache/api_app/Dockerfile @@ -0,0 +1,10 @@ +FROM python:3.12.1-slim +WORKDIR /app +EXPOSE 8080 +COPY requirements.txt ./ +# Устанавливаем зависимости python не пересобирая их +RUN pip install --no-cache --no-cache-dir -r requirements.txt +# Копирование кода приложения +COPY app.py /app/ +ENTRYPOINT ["uvicorn"] +CMD ["app:app", "--host", "0.0.0.0", "--port", "8080"] diff --git a/sharding-repl-cache/api_app/app.py b/sharding-repl-cache/api_app/app.py new file mode 100644 index 00000000..9b19c017 --- /dev/null +++ b/sharding-repl-cache/api_app/app.py @@ -0,0 +1,192 @@ +import json +import logging +import os +import time +from typing import List, Optional + +import motor.motor_asyncio +from bson import ObjectId +from fastapi import Body, FastAPI, HTTPException, status +from fastapi_cache import FastAPICache +from fastapi_cache.backends.redis import RedisBackend +from fastapi_cache.decorator import cache +from logmiddleware import RouterLoggingMiddleware, logging_config +from pydantic import BaseModel, ConfigDict, EmailStr, Field +from pydantic.functional_validators import BeforeValidator +from pymongo import errors +from redis import asyncio as aioredis +from typing_extensions import Annotated + +# Configure JSON logging +logging.config.dictConfig(logging_config) +logger = logging.getLogger(__name__) + +app = FastAPI() +app.add_middleware( + RouterLoggingMiddleware, + logger=logger, +) + +DATABASE_URL = os.environ["MONGODB_URL"] +DATABASE_NAME = os.environ["MONGODB_DATABASE_NAME"] +REDIS_URL = os.getenv("REDIS_URL", None) + + +def nocache(*args, **kwargs): + def decorator(func): + return func + + return decorator + + +if REDIS_URL: + cache = cache +else: + cache = nocache + + +client = motor.motor_asyncio.AsyncIOMotorClient(DATABASE_URL) +db = client[DATABASE_NAME] + +# Represents an ObjectId field in the database. +# It will be represented as a `str` on the model so that it can be serialized to JSON. +PyObjectId = Annotated[str, BeforeValidator(str)] + + +@app.on_event("startup") +async def startup(): + if REDIS_URL: + redis = aioredis.from_url(REDIS_URL, encoding="utf8", decode_responses=True) + FastAPICache.init(RedisBackend(redis), prefix="api:cache") + + +class UserModel(BaseModel): + """ + Container for a single user record. + """ + + id: Optional[PyObjectId] = Field(alias="_id", default=None) + age: int = Field(...) + name: str = Field(...) + + +class UserCollection(BaseModel): + """ + A container holding a list of `UserModel` instances. + """ + + users: List[UserModel] + + +@app.get("/") +async def root(): + collection_names = await db.list_collection_names() + collections = {} + for collection_name in collection_names: + collection = db.get_collection(collection_name) + collections[collection_name] = { + "documents_count": await collection.count_documents({}) + } + try: + replica_status = await client.admin.command("replSetGetStatus") + replica_status = json.dumps(replica_status, indent=2, default=str) + except errors.OperationFailure: + replica_status = "No Replicas" + + topology_description = client.topology_description + read_preference = client.client_options.read_preference + topology_type = topology_description.topology_type_name + replicaset_name = topology_description.replica_set_name + + shards = None + if topology_type == "Sharded": + shards_list = await client.admin.command("listShards") + shards = {} + for shard in shards_list.get("shards", {}): + shards[shard["_id"]] = shard["host"] + + cache_enabled = False + if REDIS_URL: + cache_enabled = FastAPICache.get_enable() + + return { + "mongo_topology_type": topology_type, + "mongo_replicaset_name": replicaset_name, + "mongo_db": DATABASE_NAME, + "read_preference": str(read_preference), + "mongo_nodes": client.nodes, + "mongo_primary_host": client.primary, + "mongo_secondary_hosts": client.secondaries, + "mongo_address": client.address, + "mongo_is_primary": client.is_primary, + "mongo_is_mongos": client.is_mongos, + "collections": collections, + "shards": shards, + "cache_enabled": cache_enabled, + "status": "OK", + } + + +@app.get("/{collection_name}/count") +async def collection_count(collection_name: str): + collection = db.get_collection(collection_name) + items_count = await collection.count_documents({}) + # status = await client.admin.command('replSetGetStatus') + # import ipdb; ipdb.set_trace() + return {"status": "OK", "mongo_db": DATABASE_NAME, "items_count": items_count} + + +@app.get( + "/{collection_name}/users", + response_description="List all users", + response_model=UserCollection, + response_model_by_alias=False, +) +@cache(expire=60 * 1) +async def list_users(collection_name: str): + """ + List all of the user data in the database. + The response is unpaginated and limited to 1000 results. + """ + time.sleep(1) + collection = db.get_collection(collection_name) + return UserCollection(users=await collection.find().to_list(1000)) + + +@app.get( + "/{collection_name}/users/{name}", + response_description="Get a single user", + response_model=UserModel, + response_model_by_alias=False, +) +async def show_user(collection_name: str, name: str): + """ + Get the record for a specific user, looked up by `name`. + """ + + collection = db.get_collection(collection_name) + if (user := await collection.find_one({"name": name})) is not None: + return user + + raise HTTPException(status_code=404, detail=f"User {name} not found") + + +@app.post( + "/{collection_name}/users", + response_description="Add new user", + response_model=UserModel, + status_code=status.HTTP_201_CREATED, + response_model_by_alias=False, +) +async def create_user(collection_name: str, user: UserModel = Body(...)): + """ + Insert a new user record. + + A unique `id` will be created and provided in the response. + """ + collection = db.get_collection(collection_name) + new_user = await collection.insert_one( + user.model_dump(by_alias=True, exclude=["id"]) + ) + created_user = await collection.find_one({"_id": new_user.inserted_id}) + return created_user diff --git a/sharding-repl-cache/api_app/requirements.txt b/sharding-repl-cache/api_app/requirements.txt new file mode 100644 index 00000000..61b29b87 --- /dev/null +++ b/sharding-repl-cache/api_app/requirements.txt @@ -0,0 +1,6 @@ +fastapi==0.110.2 +uvicorn[standard]==0.29.0 +motor==3.5.3 +redis==4.4.2 +fastapi-cache2==0.2.0 +logmiddleware==0.0.4 \ No newline at end of file diff --git a/sharding-repl-cache/compose.yaml b/sharding-repl-cache/compose.yaml new file mode 100644 index 00000000..a58e20d3 --- /dev/null +++ b/sharding-repl-cache/compose.yaml @@ -0,0 +1,253 @@ +name: sharding-repl-cache +services: + + configSrv: + image: mongo:latest + container_name: configSrv + restart: always + ports: + - "27017:27017" + networks: + app-network: + ipv4_address: 173.17.0.10 + volumes: + - config-data:/data/db + command: + [ + "--configsvr", + "--replSet", + "config_server", + "--bind_ip_all", + "--port", + "27017" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + + shard1_db1: + image: mongo:latest + container_name: shard1_db1 + restart: always + ports: + - "27018:27018" + networks: + app-network: + ipv4_address: 173.17.0.9 + volumes: + - shard1_db1-data:/data/db + command: + [ + "--shardsvr", + "--replSet", + "shard1", + "--bind_ip_all", + "--port", + "27018" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + + shard1_db2: + image: mongo:latest + container_name: shard1_db2 + restart: always + ports: + - "27028:27028" + networks: + app-network: + ipv4_address: 173.17.0.12 + volumes: + - shard1_db2-data:/data/db + command: + [ + "--shardsvr", + "--replSet", + "shard1", + "--bind_ip_all", + "--port", + "27028" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + + + shard1_db3: + image: mongo:latest + container_name: shard1_db3 + restart: always + ports: + - "27038:27038" + networks: + app-network: + ipv4_address: 173.17.0.13 + volumes: + - shard1_db3-data:/data/db + command: + [ + "--shardsvr", + "--replSet", + "shard1", + "--bind_ip_all", + "--port", + "27038" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + + + shard2_db1: + image: mongo:latest + container_name: shard2_db1 + restart: always + ports: + - "27019:27019" + networks: + app-network: + ipv4_address: 173.17.0.8 + volumes: + - shard2_db1-data:/data/db + command: + [ + "--shardsvr", + "--replSet", + "shard2", + "--bind_ip_all", + "--port", + "27019" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + + shard2_db2: + image: mongo:latest + container_name: shard2_db2 + restart: always + ports: + - "27029:27029" + networks: + app-network: + ipv4_address: 173.17.0.15 + volumes: + - shard2_db2-data:/data/db + command: + [ + "--shardsvr", + "--replSet", + "shard2", + "--bind_ip_all", + "--port", + "27029" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + + shard2_db3: + image: mongo:latest + container_name: shard2_db3 + restart: always + ports: + - "27039:27039" + networks: + app-network: + ipv4_address: 173.17.0.16 + volumes: + - shard2_db3-data:/data/db + command: + [ + "--shardsvr", + "--replSet", + "shard2", + "--bind_ip_all", + "--port", + "27039" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + + mongos_router: + image: mongo:latest + container_name: mongos_router + restart: always + ports: + - "27020:27020" + networks: + app-network: + ipv4_address: 173.17.0.7 + command: + [ + "mongos", + "--configdb", + "config_server/configSrv:27017", + "--bind_ip_all", + "--port", + "27020" + ] + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + start_period: 10s + + + redis_1: + image: "redis:latest" + container_name: redis_1 + ports: + - "6379" + volumes: + - redis_1_data:/data + networks: + app-network: + ipv4_address: 173.17.0.14 + + + pymongo_api: + container_name: pymongo_api + build: + context: api_app + dockerfile: Dockerfile + image: kazhem/pymongo_api:1.0.0 + depends_on: + - mongos_router + ports: + - 8080:8080 + networks: + app-network: + ipv4_address: 173.17.0.11 + environment: + MONGODB_URL: "mongodb://mongos_router:27020" + MONGODB_DATABASE_NAME: "somedb" + REDIS_URL: "redis://redis_1:6379" + + +networks: + app-network: + driver: bridge + ipam: + driver: default + config: + - subnet: 173.17.0.0/16 + + +volumes: + config-data: + shard1_db1-data: + shard1_db2-data: + shard1_db3-data: + shard2_db1-data: + shard2_db2-data: + shard2_db3-data: + redis_1_data: {} diff --git a/sharding-repl-cache/scripts/db-drop.sh b/sharding-repl-cache/scripts/db-drop.sh new file mode 100644 index 00000000..b4292c4f --- /dev/null +++ b/sharding-repl-cache/scripts/db-drop.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +### +# Очистка бд +### + +docker compose exec -T mongos_router mongosh --port 27020 --quiet < Date: Sat, 23 Nov 2024 12:54:51 +0700 Subject: [PATCH 6/6] Add scheme.drawio, edit README.md --- README.md | 137 ++++++++++++++- scheme.drawio | 455 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 585 insertions(+), 7 deletions(-) create mode 100644 scheme.drawio diff --git a/README.md b/README.md index b6ddb826..631fb798 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,88 @@ -# pymongo-api +# Репозиторий pymongo-api -## Как запустить +Репозиторий содержит стуктуру директорий c вариантами реализаций приложения и структурную схему cheme.drawio. +Результатом запуска является развертывание сервиса управления данными на http://:8080. -Запускаем mongodb и приложение +## Задание1. Проектирование +На основе шаблона необходимо создать схему итогового решения. + +1. Схема с шардированием с двумя шардами MongoDB https://drive.google.com/file/d/1Yd0LxC7VE9Yqa9HL5vOvSke4Psv8tViy/view?usp=sharing +2. Репликацию MongoDB https://drive.google.com/file/d/1DtlZUV0B0FKdAe5qVzqNcSVcwjM4ETIT/view?usp=sharing +3. Кэширование с Redis https://drive.google.com/file/d/1h76DW9y2f82uBVmRDcxmS2nfHk3ApoDa/view?usp=sharing + +Для каждого этапа представлены отдельные схемы draw.io + + +## Заданиея 2,3 и 4. Реализация шардирования, репликации и кэширования + +Репозиторий содержит стуктуру директорий, в каждой из которых находятся исполняемые файлы запуска compose.yaml приложения, +в соответствии с заданиями 2,3 и 4. + +```sh +/mongo-sharding + /scripts + compose.yaml +/mongo-sharding-repl + /scripts + compose.yaml +/sharding-repl-cache + /scripts + compose.yaml +``` +В соотвествующих директориях /scrips находятся исполняемые файлы для запуска приложений. + +### Как запустить + +Необходимо перейти в директорию с соответвующим compose.yaml файлом в корне + +```shell +cd sharding-repl-cache +``` + +Запустить формирование контейнеров ```shell docker compose up -d ``` -Заполняем mongodb данными +Инициазировать инстансы, запустив скрипт ```shell ./scripts/mongo-init.sh ``` -## Как проверить +Наполнить данными и вывести результат в консоль, запустить скрипт + +```shell +./scripts/load-output.sh +``` + +Результат запуска в консоли + +```shell +[direct: mongos] somedb> NOTE: helloDoc count=1000 +[direct: mongos] somedb> shard1 [direct: primary] test> OUTPUT: shard1 repl count=3 +shard1 [direct: primary] test> shard1 [direct: primary] test> switched to db somedb +shard1 [direct: primary] somedb> NOTE: shard1_db1 doc count=492 +shard1 [direct: primary] somedb> shard1 [direct: secondary] test> switched to db somedb +shard1 [direct: secondary] somedb> NOTE: shard1_db2 doc count=492 +shard1 [direct: secondary] somedb> shard1 [direct: secondary] test> switched to db somedb +shard1 [direct: secondary] somedb> NOTE: shard1_db3 doc count=492 +shard1 [direct: secondary] somedb> shard2 [direct: primary] test> +shard2 [direct: primary] test> OUTPUT: shard2 repl count=3 +shard2 [direct: primary] test> shard2 [direct: primary] test> switched to db somedb +shard2 [direct: primary] somedb> NOTE: shard2_db1 doc count=508 +shard2 [direct: primary] somedb> shard2 [direct: secondary] test> switched to db somedb +shard2 [direct: secondary] somedb> NOTE: shard2_db2 doc count=508 +shard2 [direct: secondary] somedb> shard2 [direct: secondary] test> switched to db somedb +shard2 [direct: secondary] somedb> NOTE: shard2_db3 doc count=508 +``` + +При необходимости очистить базу + +```shell +./scripts/db-drop.sh +``` ### Если вы запускаете проект на локальной машине @@ -30,6 +98,61 @@ curl --silent http://ifconfig.me Откройте в браузере http://:8080 -## Доступные эндпоинты +```sh +{ + "mongo_topology_type": "Sharded", + "mongo_replicaset_name": null, + "mongo_db": "somedb", + "read_preference": "Primary()", + "mongo_nodes": [ + [ + "mongos_router", + 27020] + ], + "mongo_primary_host": null, + "mongo_secondary_hosts": [], + "mongo_address": [ + "mongos_router", + 27020], + "mongo_is_primary": true, + "mongo_is_mongos": true, + "collections": { + "helloDoc": { + "documents_count": 1000 + } + }, + "shards": { + "shard1": "shard1/shard1_db1:27018,shard1_db2:27028,shard1_db3:27038", + "shard2": "shard2/shard2_db1:27019,shard2_db2:27029,shard2_db3:27039" + }, + "cache_enabled": true, + "status": "OK" +} +``` + +### Доступные эндпоинты + +Список доступных эндпоинтов, swagger http://:8080/docs + +## Задание5. Service Discovery и балансировка с API Gateway + +В результате выполнения задания составлен вариант схемы с горизонтальным масштабированием приложения. + +https://drive.google.com/file/d/1jZaGPW_JHP0NUKkqXiOT4F8BF367rrJo/view?usp=sharing + +Добавлен сервис API Gateway для балансировки и Consul для реализации Service Discovery. + + +## Задание6. Service Discovery и балансировка с API Gateway +В результате выполнения задания составлен вариант схемы, на котором реализовано использование CDN. + +https://drive.google.com/file/d/19FNsYijtTVUUQEwlMgfLVaONL_f8pUwd/view?usp=sharing + +Добавлен облачный сервис CNN. + + +## Итоговая схема + +Итоговая структурна схема https://drive.google.com/file/d/114Zqvy8q64Qslb2dnt4OUSmorduqFUZ5/view?usp=sharing -Список доступных эндпоинтов, swagger http://:8080/docs \ No newline at end of file +представлена в файле cheme.drawio в корне приложения. diff --git a/scheme.drawio b/scheme.drawio new file mode 100644 index 00000000..76319f5f --- /dev/null +++ b/scheme.drawio @@ -0,0 +1,455 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +