diff --git a/.env b/.env index 2a3f64b..5089662 100644 --- a/.env +++ b/.env @@ -1,7 +1,8 @@ DB_USER=postgres DB_PASSWORD=password DB_HOST=localhost -DB_PORT=5432 +; DB_PORT=5432 +DB_PORT=20111 DB_NAME=test DATABASE=SuperService POSTGRES_USER=UserSuperService diff --git a/db/csv_to_db.py b/db/csv_to_db.py index 691251d..dde3e95 100644 --- a/db/csv_to_db.py +++ b/db/csv_to_db.py @@ -3,22 +3,42 @@ from sqlalchemy.ext.asyncio import create_async_engine, async_sessionmaker from sqlalchemy import insert from db.config import settings from db.models.base import Base +from db.models.ch_experimentdb_experiment_data_model import ChExperimentDBExperimentData from db.models.experiment_data_model import ExperimentData from db.models.experiment_parameters_model import ExperimentParameters -from db.models.load_parameters_model import LoadParameters +from db.models.load_parameters_model import LoadParameters from db.models.recycling_parameters_model import RecyclingParameters import asyncio -engine = create_async_engine(url=settings.db_url_asyncpg, echo=True) +engine = create_async_engine(url=settings.db_url_asyncpg_docker, echo=True) async_session = async_sessionmaker(engine) +def add_ids_in_csv(file: str): + try: + df = pd.read_csv(file) + df.insert(0, 'id', pd.Series(range(1, len(d) + 1))) + df.to_csv(file, index=False) + except Exception as e: + print(f'Exception!! {e}') + + +def print_headers_and_types(file: str): + df = pd.read_csv(file) + headers = df.columns.tolist() + print(headers) + + for header in headers: + column_type = df[header].dtype + print(column_type) + async def create_all_tables(): async with engine.begin() as conn: await conn.run_sync(Base.metadata.create_all) + async def drop_all_tables(): async with engine.begin() as conn: await conn.run_sync(Base.metadata.drop_all) @@ -37,20 +57,27 @@ async def load_data_to_db(file: str, model_class): await session.commit() -async def main(): - # await drop_all_tables() - # await create_all_tables() - await load_data_to_db('./files/experiment_data.csv', ExperimentData) - # await load_data_to_db('./files/load_parameters.csv', LoadParameters) - # await load_data_to_db('./files/recycling_parameters.csv', RecyclingParameters) - # await load_data_to_db('./files/experiment_parameters.csv', ExperimentParameters) +async def test(file: str, model_class): + async with async_session() as session: + df = pd.read_csv(file) + df.dropna(inplace=True) + df = df.head(20) + data_records = df.to_dict(orient='records') + stmt = insert(model_class).values(data_records) + await session.execute(stmt) -if __name__ == '__main__': - asyncio.run(main()) - # df = pd.read_csv('./files/experiment_data.csv') - # headers = df.columns.tolist() - # print(headers) - # - # for header in headers: - # column_type = df[header].dtype - # print(column_type) + await session.commit() + + +async def csv_to_db(): + await drop_all_tables() + await create_all_tables() + await load_data_to_db('./files/ch_experimentdb_experiment_data.csv', ChExperimentDBExperimentData) + await test('./files/experiment_data.csv', ExperimentData) + await load_data_to_db('./files/load_parameters.csv', LoadParameters) + await load_data_to_db('./files/recycling_parameters.csv', RecyclingParameters) + await load_data_to_db('./files/experiment_parameters.csv', ExperimentParameters) + + + +# asyncio.run(csv_to_db()) diff --git a/db/files/ch_experimentdb_experiment_data.csv b/db/files/ch_experimentdb_experiment_data.csv index 833fd48..5a0b8a6 100644 --- a/db/files/ch_experimentdb_experiment_data.csv +++ b/db/files/ch_experimentdb_experiment_data.csv @@ -1,11 +1,11 @@ -volume,nitrogen_oxide_emission,temperature,co_fraction,co2_fraction,x,y,z,file_id -0.00000012501800906768677,0.0000000002923801478004699,1546.8502257117334,0.0876590170721079,0.0825937575747152,0.00028629083520050004,4.250519128266181,4.23689607830385,10bf3b81cf73056684c0041f6adc5a35881bbeca2ba0e8bbc7d827c3ba058a3d -0.0000001262767070227717,0.00000000027744029871169775,1546.5278876054947,0.0876535741398523,0.0826072611209551,0.0002970890883859,4.2633663477653,4.2118739518585935,10bf3b81cf73056684c0041f6adc5a35881bbeca2ba0e8bbc7d827c3ba058a3d -0.0000005215560244805981,0.00000000028009892259604384,1547.007375203737,0.087598352075182,0.0826205755990879,0.0023288861534884,4.258002712932174,4.224854019469968,10bf3b81cf73056684c0041f6adc5a35881bbeca2ba0e8bbc7d827c3ba058a3d -0.0000001259521373287687,0.00000000027568361021801806,1544.7738826604286,0.0879623682225424,0.0825022367344128,0.0003061405654724,4.294951651093852,4.228359039356472,10bf3b81cf73056684c0041f6adc5a35881bbeca2ba0e8bbc7d827c3ba058a3d -0.00000006712047787057572,0.00000000027478227962893084,1543.303419205242,0.0881230454580786,0.0824746033833566,0.0004276277312526001,4.301939013567143,4.203422399444153,10bf3b81cf73056684c0041f6adc5a35881bbeca2ba0e8bbc7d827c3ba058a3d -0.0000003982683542080528,0.00000000027289594477188307,1544.1211163717342,0.0880178768655007,0.0824974031307205,0.0025840772927808,4.2895374677272695,4.241641279196477,10bf3b81cf73056684c0041f6adc5a35881bbeca2ba0e8bbc7d827c3ba058a3d -0.00000013145014912289965,0.0000000002729991676088085,1543.9585181296568,0.0881077612277477,0.0824593957518346,0.0003192230503286001,4.282153424064967,4.253165462380294,10bf3b81cf73056684c0041f6adc5a35881bbeca2ba0e8bbc7d827c3ba058a3d -0.0000012711512153309188,0.0000000002799785902700662,1545.337937815918,0.0878016656787745,0.0825684205234254,0.0031927006255012,4.279075221963268,4.242926141081111,10bf3b81cf73056684c0041f6adc5a35881bbeca2ba0e8bbc7d827c3ba058a3d -0.0000005002643798196125,0.00000000029077782102827164,1546.1911065379516,0.0876446332442777,0.0826230869307894,0.0012912004540912001,4.259093754350848,4.2392143443160855,10bf3b81cf73056684c0041f6adc5a35881bbeca2ba0e8bbc7d827c3ba058a3d -0.00000043725159034908665,0.0000000002852153061274629,1545.394093220484,0.0877377183636819,0.0826003036520615,0.0010234056556132002,4.268539978944442,4.241752821543271,10bf3b81cf73056684c0041f6adc5a35881bbeca2ba0e8bbc7d827c3ba058a3d +id,volume,nitrogen_oxide_emission,temperature,co_fraction,co2_fraction,x,y,z,file_id +1,1.25018009e-07,2.923801e-10,1546.8502257117334,0.0876590170721079,0.0825937575747152,0.0002862908352005,4.250519128266181,4.23689607830385,10bf3b81cf73056684c0041f6adc5a35881bbeca2ba0e8bbc7d827c3ba058a3d +2,1.26276707e-07,2.774402e-10,1546.5278876054947,0.0876535741398523,0.0826072611209551,0.0002970890883859,4.2633663477653,4.2118739518585935,10bf3b81cf73056684c0041f6adc5a35881bbeca2ba0e8bbc7d827c3ba058a3d +3,5.215560244e-07,2.800989e-10,1547.007375203737,0.087598352075182,0.0826205755990879,0.0023288861534884,4.258002712932174,4.224854019469968,10bf3b81cf73056684c0041f6adc5a35881bbeca2ba0e8bbc7d827c3ba058a3d +4,1.259521373e-07,2.756836e-10,1544.7738826604286,0.0879623682225424,0.0825022367344128,0.0003061405654724,4.294951651093852,4.228359039356472,10bf3b81cf73056684c0041f6adc5a35881bbeca2ba0e8bbc7d827c3ba058a3d +5,6.71204778e-08,2.747822e-10,1543.303419205242,0.0881230454580786,0.0824746033833566,0.0004276277312526,4.301939013567143,4.203422399444153,10bf3b81cf73056684c0041f6adc5a35881bbeca2ba0e8bbc7d827c3ba058a3d +6,3.982683542e-07,2.728959e-10,1544.1211163717342,0.0880178768655007,0.0824974031307205,0.0025840772927808,4.2895374677272695,4.241641279196477,10bf3b81cf73056684c0041f6adc5a35881bbeca2ba0e8bbc7d827c3ba058a3d +7,1.314501491e-07,2.729991e-10,1543.9585181296568,0.0881077612277477,0.0824593957518346,0.0003192230503286,4.282153424064967,4.253165462380294,10bf3b81cf73056684c0041f6adc5a35881bbeca2ba0e8bbc7d827c3ba058a3d +8,1.2711512153e-06,2.799785e-10,1545.337937815918,0.0878016656787745,0.0825684205234254,0.0031927006255012,4.279075221963268,4.242926141081111,10bf3b81cf73056684c0041f6adc5a35881bbeca2ba0e8bbc7d827c3ba058a3d +9,5.002643798e-07,2.907778e-10,1546.1911065379516,0.0876446332442777,0.0826230869307894,0.0012912004540912,4.259093754350848,4.2392143443160855,10bf3b81cf73056684c0041f6adc5a35881bbeca2ba0e8bbc7d827c3ba058a3d +10,4.372515903e-07,2.852153e-10,1545.394093220484,0.0877377183636819,0.0826003036520615,0.0010234056556132,4.268539978944442,4.241752821543271,10bf3b81cf73056684c0041f6adc5a35881bbeca2ba0e8bbc7d827c3ba058a3d diff --git a/db/models/base.py b/db/models/base.py index 1c2dcc4..65a112f 100644 --- a/db/models/base.py +++ b/db/models/base.py @@ -1,5 +1,8 @@ -from sqlalchemy.orm import DeclarativeBase +from typing import Annotated +from sqlalchemy.orm import DeclarativeBase, mapped_column + +int_pk_incr = Annotated[int, mapped_column(primary_key=True, autoincrement=True)] class Base(DeclarativeBase): pass \ No newline at end of file diff --git a/db/models/ch_experimentdb_experiment_data_model.py b/db/models/ch_experimentdb_experiment_data_model.py index 832d4bd..a234236 100644 --- a/db/models/ch_experimentdb_experiment_data_model.py +++ b/db/models/ch_experimentdb_experiment_data_model.py @@ -1,10 +1,20 @@ -from db.models.base import Base -from sqlalchemy.orm import Mapped, mapped_column +from db.models.base import Base, int_pk_incr +from sqlalchemy.orm import Mapped + class ChExperimentDBExperimentData(Base): - __tablename__ = 'ChExperimentDBExperimentData' + __tablename__ = 'ch_experimentdb_experiment_data' - id: Mapped[int] = mapped_column(primary_key=True) + id: Mapped[int_pk_incr] + volume: Mapped[float] + nitrogen_oxide_emission: Mapped[float] + temperature: Mapped[float] + co_fraction: Mapped[float] + co2_fraction: Mapped[float] + x: Mapped[float] + y: Mapped[float] + z: Mapped[float] + file_id: Mapped[str] def __repr__(self): - return f"" \ No newline at end of file + return f"" diff --git a/db/models/experiment_data_model.py b/db/models/experiment_data_model.py index 0d78ab4..3bd8eb7 100644 --- a/db/models/experiment_data_model.py +++ b/db/models/experiment_data_model.py @@ -1,12 +1,12 @@ from typing import Optional -from db.models.base import Base +from db.models.base import Base, int_pk_incr from sqlalchemy.orm import Mapped, mapped_column class ExperimentData(Base): __tablename__ = 'experiment_data' - id: Mapped[int] = mapped_column(primary_key=True) + id: Mapped[int_pk_incr] direction: Mapped[float] temperature: Mapped[float] nox: Mapped[float] diff --git a/db/models/experiment_parameters_model.py b/db/models/experiment_parameters_model.py index 0d342fb..5f3d0e2 100644 --- a/db/models/experiment_parameters_model.py +++ b/db/models/experiment_parameters_model.py @@ -2,18 +2,17 @@ from typing import Optional from sqlalchemy import ForeignKey -from db.models.base import Base +from db.models.base import Base, int_pk_incr from sqlalchemy.orm import Mapped, mapped_column class ExperimentParameters(Base): __tablename__ = 'experiment_parameters' - id: Mapped[int] = mapped_column(primary_key=True) + id: Mapped[int_pk_incr] outer_blades_count: Mapped[int] outer_blades_length: Mapped[float] outer_blades_angle: Mapped[float] middle_blades_count: Mapped[int] - # load_id: Mapped[int] load_id: Mapped[Optional[int]] = mapped_column(ForeignKey('load_parameters.id', ondelete='SET NULL')) recycling_id: Mapped[Optional[int]] = mapped_column(ForeignKey('recycling_parameters.id', ondelete='SET NULL')) experiment_hash: Mapped[str] diff --git a/db/models/load_parameters_model.py b/db/models/load_parameters_model.py index cb90a77..16d215d 100644 --- a/db/models/load_parameters_model.py +++ b/db/models/load_parameters_model.py @@ -1,14 +1,15 @@ -from db.models.base import Base -from sqlalchemy.orm import Mapped, mapped_column +from db.models.base import Base, int_pk_incr +from sqlalchemy.orm import Mapped + class LoadParameters(Base): __tablename__ = 'load_parameters' - id: Mapped[int] = mapped_column(primary_key=True) + id: Mapped[int_pk_incr] load: Mapped[int] primary_air_consumption: Mapped[float] secondary_air_consumption: Mapped[float] gas_inlet_consumption: Mapped[float] def __repr__(self): - return f"" \ No newline at end of file + return f"" diff --git a/db/models/recycling_parameters_model.py b/db/models/recycling_parameters_model.py index 6b8e6e7..de1f8ca 100644 --- a/db/models/recycling_parameters_model.py +++ b/db/models/recycling_parameters_model.py @@ -2,13 +2,13 @@ from typing import Optional from sqlalchemy import ForeignKey -from db.models.base import Base +from db.models.base import Base, int_pk_incr from sqlalchemy.orm import Mapped, mapped_column class RecyclingParameters(Base): __tablename__ = 'recycling_parameters' - id: Mapped[int] = mapped_column(primary_key=True) + id: Mapped[int_pk_incr] load_id: Mapped[Optional[int]] = mapped_column(ForeignKey('load_parameters.id', ondelete='SET NULL')) recycling_level: Mapped[int] co2: Mapped[float] diff --git a/main.py b/main.py index 7031cd1..f8f9062 100644 --- a/main.py +++ b/main.py @@ -1,6 +1,7 @@ from fastapi import FastAPI, HTTPException from pydantic import BaseModel from new_experiment_planner import run_experiment # Импортируем функцию из твоего скрипта +from db.csv_to_db import csv_to_db app = FastAPI() @@ -34,7 +35,7 @@ def run_experiment_api(params: ExperimentParameters): @app.post('/init_db_data') def init_db_data(): try: - + csv_to_db() return {"status": "success", "message": "Experiment started successfully."} except Exception as e: raise HTTPException(status_code=500, detail=f"An error occurred: {str(e)}")