Skip to content

SimplyCommon API Reference

database

SimulationTable

Bases: SQLModel

The table in the database for a simulation item. Name is Optional as we don't have to name each runs. This table is linked to the artifact table. Each runs has its own artefact table.

Source code in simplycommon/database.py 
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
class SimulationTable(SQLModel, table=True):
    """
    The table in the database for a simulation item. Name is Optional as we don't have to name each runs. This table is linked to the artifact table. Each runs has its own artefact table.
    """

    id: Optional[int] = Field(default=None, primary_key=True)
    uuid: str
    name: Optional[str]
    date: Optional[str]
    parameters: Dict[str, Any] = Field(default={}, sa_column=Column(JSON))
    results: Dict[str, Any] = Field(default={}, sa_column=Column(JSON))
    meta_data: Dict[str, Any] = Field(default={}, sa_column=Column(JSON))
    tag: Optional[Dict[str, Any]] = Field(default={}, sa_column=Column(JSON))
    note: Optional[Dict[str, Any]] = Field(default={}, sa_column=Column(JSON))
    artifacts: List["ArtifactTable"] = Relationship(back_populates="simulation")

check_hashes_exists(input_hash, engine)

Returns the path and hash of a artifact if it already exists is the db to save space.

Source code in simplycommon/database.py 
456
457
458
459
460
461
462
463
464
465
466
def check_hashes_exists(input_hash, engine):
    """
    Returns the path and hash of a artifact if it already exists is the db to save space.
    """
    with Session(engine) as session:
        statement = select(ArtifactTable).where(ArtifactTable.hash == input_hash)
        results = session.exec(statement).first()
        if results is None:
            return results
        else:
            return results.path, results.hash

sim_to_db(run, session)

Dump a simulation item into the simply database.

Source code in simplycommon/database.py 
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
def sim_to_db(run, session):
    """
    Dump a simulation item into the simply database.
    """
    clean_results = {}
    # We create a list to hold all the artifacts for this specific run
    run_artifacts: List[ArtifactTable] = []

    for res_name, res_item in run.results.items():
        if not res_item.is_artifact:
            # CASE 1: It's just a value or a pointer (not a file)
            clean_results[res_name] = res_item.value
        else:
            # CASE 2: IT IS AN ARTIFACT!
            # We create the ArtifactTable row object
            new_artifact = ArtifactTable(
                name=res_name,
                hash=res_item.hsh,
                path=res_item.value,
            )
            run_artifacts.append(new_artifact)

    # Create the Simulation Database Row
    table = SimulationTable(
        uuid=run.run_id,
        name=run.run_name,
        date=run.date,
        parameters=run.parameters,
        results=clean_results,
        meta_data=run.metadata,
        tag=run.tags,
        note=run.notes,
        artifacts=run_artifacts,
    )

    session.add(table)

    # When we hit commit, SQLModel will:
    # 1. Save the SimulationTable and get its new 'id'
    # 2. Automatically assign that 'id' to the 'run_id' of every artifact in the list
    # 3. Save all the ArtifactTables to the database.
    session.commit()

    return table

sims_to_db(simulations, session)

Dump an entire sim_dict to the database.

Source code in simplycommon/database.py 
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
def sims_to_db(simulations: dict, session):
    """
    Dump an entire sim_dict to the database.
    """
    saved_tables = []

    for run_id, run in simulations.items():
        clean_results = {}
        # We create a list to hold all the artifacts for this specific run
        run_artifacts: List[ArtifactTable] = []

        for res_name, res_item in run.results.items():
            if not res_item.is_artifact:
                # CASE 1: It's just a value or a pointer (not a file)
                clean_results[res_name] = res_item.value
            else:
                # CASE 2: IT IS AN ARTIFACT!
                # We create the ArtifactTable row object
                new_artifact = ArtifactTable(
                    name=res_name,
                    hash=res_item.hsh,
                    path=res_item.value,
                )
                run_artifacts.append(new_artifact)

        # Create the Simulation Database Row
        table = SimulationTable(
            uuid=run_id,
            name=run.run_name,
            date=run.date,
            parameters=run.parameters,
            results=clean_results,
            meta_data=run.metadata,
            tag=run.tags,
            note=run.notes,
            artifacts=run_artifacts,
        )

        session.add(table)
        saved_tables.append(table)

    # When we hit commit, SQLModel will:
    # 1. Save the SimulationTable and get its new 'id'
    # 2. Automatically assign that 'id' to the 'run_id' of every artifact in the list
    # 3. Save all the ArtifactTables to the database.
    session.commit()

    return saved_tables

misc

Context

A runtime object is a condensed of metadata to give an overview of a run.

Source code in simplycommon/misc.py 
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
class Context:
    """
    A runtime object is a condensed of metadata to give an overview of a run.
    """

    def __init__(self, engine, run_id):
        self.run_id = run_id
        self.meta_data = select_metadata(engine, run_id)[0]
        self.parameter_length = len(select_param(engine, run_id))
        self.result_length = len(select_result(engine, run_id))
        self.runtime = self.meta_data["simply.runtime"]
        self.platform = self.meta_data["simply.language"]
        self.date_time = self.meta_data["simply.date"]
        if self.platform == "python":
            self.main_script_source = self.meta_data["simply.python.main_script_source"]

simulation

Simulation

Source code in simplycommon/simulation.py 
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
class Simulation:
    def __init__(self, run_id, run_name, project_path):
        self.run_id = run_id
        self.run_name = run_name
        self.parameters: Dict[str, Any] = {}
        self.results: Dict[str, ResultItem] = {}
        self.metadata: Dict[str, Any] = {}
        self.tags: Dict[str, Any] = {}
        self.notes: Dict[str, Any] = {}
        self.envh = RunEnvironmentHandler(run_id, project_path)
        self.glob = self.envh.create_glob()

        self.check_name()
        self.set_date()

    def check_name(self):
        if self.run_name == "" or self.run_name is None:
            self.run_name = generate_name() 

    def set_date(self):
        self.date = datetime.now().strftime("%Y-%m-%d-%H:%M:%S")


    """
    Saving functions
    """

    def log_param(self, name: str, value: Any):
        # TODO: Add overwrite check if needed
        self.parameters[name] = value

    def add_metadata(self, name: str, value: Any):
        self.metadata[name] = value

    def add_note(self, name: str, value: Any):
        self.notes[name] = value

    def add_tag(self, name: str, value: Any):
        self.tags[name] = value

    def log_result(self, name, value_dict):
        """
        The save a result command can work in a few different ways. First the api provides two ways of
        saving result. Either applying result(obj, id=None) on one single result. It will log in the
        database the entry with its artifact it should point to and save the result by default in the
        hdfs5 blob. On the other hand the command addResult(result, id=None, path=None, save_result=False)
        allow more flexibility to the user. Alowing to copy artefic once save by the user, only log file in
        the db without copying or saving a dic of result in the hdfs matrix.

        However this functions needs to be tested in the case of saving a large quantity of data (Future problems)

        Note that for now this function need the environment handler and the project manager. They should be created per
        project and handle as globar variables accessed by each object simulation to manipulate the .simply dir.
        In short, this function cannot work and be tested yet.

        You cannot find path to be none and save_result to be false at the same time.
        """
        data = value_dict.get("value")
        path = value_dict.get("path")
        save_result = value_dict.get(
            "save_result", False
        )  # defaults to False if missing
        if path is not None:
            if save_result is True:
                pointer, hsh = self.envh.save_artifact(path)
                self.results[name] = ResultItem(pointer, is_artifact=True, hsh=hsh)
            else:
                hsh = get_hash(path)
                self.results[name] = ResultItem(path, hsh=hsh)
        if data is not None:
            pointer, hsh = self.glob.save(name, data)
            self.results[name] = ResultItem(pointer, hsh=hsh)

    """
    Loading functions
    """

    def _get_item(self, name, sim_dict):
        if name not in sim_dict:
            raise KeyError(f"Item '{name}' not found.")
        return sim_dict[name]

    def get_param(self, name):
        return self._get_item(name, self.parameters)

    def get_result(self, name):
        return self._get_item(name, self.results)

    def get_metadata(self, name):
        return self._get_item(name, self.metadata)

log_result(name, value_dict)

The save a result command can work in a few different ways. First the api provides two ways of saving result. Either applying result(obj, id=None) on one single result. It will log in the database the entry with its artifact it should point to and save the result by default in the hdfs5 blob. On the other hand the command addResult(result, id=None, path=None, save_result=False) allow more flexibility to the user. Alowing to copy artefic once save by the user, only log file in the db without copying or saving a dic of result in the hdfs matrix.

However this functions needs to be tested in the case of saving a large quantity of data (Future problems)

Note that for now this function need the environment handler and the project manager. They should be created per project and handle as globar variables accessed by each object simulation to manipulate the .simply dir. In short, this function cannot work and be tested yet.

You cannot find path to be none and save_result to be false at the same time.

Source code in simplycommon/simulation.py 
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
def log_result(self, name, value_dict):
    """
    The save a result command can work in a few different ways. First the api provides two ways of
    saving result. Either applying result(obj, id=None) on one single result. It will log in the
    database the entry with its artifact it should point to and save the result by default in the
    hdfs5 blob. On the other hand the command addResult(result, id=None, path=None, save_result=False)
    allow more flexibility to the user. Alowing to copy artefic once save by the user, only log file in
    the db without copying or saving a dic of result in the hdfs matrix.

    However this functions needs to be tested in the case of saving a large quantity of data (Future problems)

    Note that for now this function need the environment handler and the project manager. They should be created per
    project and handle as globar variables accessed by each object simulation to manipulate the .simply dir.
    In short, this function cannot work and be tested yet.

    You cannot find path to be none and save_result to be false at the same time.
    """
    data = value_dict.get("value")
    path = value_dict.get("path")
    save_result = value_dict.get(
        "save_result", False
    )  # defaults to False if missing
    if path is not None:
        if save_result is True:
            pointer, hsh = self.envh.save_artifact(path)
            self.results[name] = ResultItem(pointer, is_artifact=True, hsh=hsh)
        else:
            hsh = get_hash(path)
            self.results[name] = ResultItem(path, hsh=hsh)
    if data is not None:
        pointer, hsh = self.glob.save(name, data)
        self.results[name] = ResultItem(pointer, hsh=hsh)