1810 fix

docs/cookbook/plugins/idmtools_item_sequence_example.ini

@@ -0,0 +1,85 @@
+[COMMON]
+# Number of threads idmtools will use for analysis and other multi-threaded activities
+max_threads = 16
+
+# How many simulations per threads during simulation creation
+sims_per_thread = 20
+
+# Maximum number of LOCAL simulation ran simultaneously
+max_local_sims = 6
+
+# Maxium number of workers processing in parallel
+max_workers = 16
+
+# Maxium batch size to retrieve simulations
+batch_size = 10
+
+id_generator = item_sequence
+
+[COMPS]
+type = COMPS
+endpoint = https://comps.idmod.org
+environment = Belegost
+priority = Lowest
+simulation_root = $COMPS_PATH(USER)\output
+node_group = emod_abcd
+num_retries = 0
+num_cores = 1
+max_workers = 16
+batch_size = 10
+exclusive = False
+
+[COMPS2]
+type = COMPS
+endpoint = https://comps2.idmod.org
+environment = Bayesian
+priority = Lowest
+simulation_root = $COMPS_PATH(USER)\output
+node_group = emod_abcd
+num_retries = 0
+num_cores = 1
+max_workers = 16
+batch_size = 10
+exclusive = False
+test1 = 1
+test2 = 2
+
+[SSMT]
+type = SSMT
+endpoint = https://comps2.idmod.org
+environment = Bayesian
+priority = Lowest
+simulation_root = $COMPS_PATH(USER)\output
+node_group = emod_abcd
+num_retries = 0
+num_cores = 1
+max_workers = 16
+batch_size = 10
+exclusive = False
+
+[Local]
+type = Local
+
+# Path to the model executable
+exe_path = \\internal.idm.ctr\IDM-Test\home\mafisher\bin\Eradication.exe
+
+[Logging]
+level = DEBUG
+console = on
+
+
+# This is a test we used to validate loading local from section block
+[Custom_Local]
+type = Local
+
+[Test]
+type = Test
+
+[BADTYPE]
+type = Bad
+
+[NOTYPE]
+
+[item_sequence]
+sequence_file = item_ids.json
+id_format_str = '{item_name}{data[item_name]:07d}'

docs/cookbook/plugins/sequential_id_example.py

@@ -0,0 +1,52 @@
+import os
+import sys
+from functools import partial
+from typing import Any, Dict
+from pathlib import Path
+from idmtools import IdmConfigParser
+from idmtools.builders import SimulationBuilder
+from idmtools.core.platform_factory import Platform
+from idmtools.entities.experiment import Experiment
+from idmtools.entities.simulation import Simulation
+from idmtools.entities.templated_simulation import TemplatedSimulations
+from idmtools_models.python.json_python_task import JSONConfiguredPythonTask
+
+# NOTE TO USER
+# You need to define your own SlurmNative configuration block before running this example
+# Please update examples/idmtools.ini accordingly
+platform = Platform('SlurmNative')
+
+with platform:
+    parser = IdmConfigParser()
+    parser._load_config_file(file_name='idmtools_item_sequence_example.ini')
+    parser.ensure_init(file_name='idmtools_item_sequence_example.ini', force=True)
+    sequence_file = Path(
+        IdmConfigParser.get_option("item_sequence", "sequence_file", 'item_sequences.json')).expanduser()
+    if sequence_file.exists():
+        sequence_file.unlink()
+
+    task = JSONConfiguredPythonTask(script_path=os.path.join("..", "..", "python_model", "inputs", "python_model_with_deps", "Assets", "model.py"),
+                                    parameters=(dict(c=0)))
+    ts = TemplatedSimulations(base_task=task)
+    builder = SimulationBuilder()
+
+    def param_update(simulation: Simulation, param: str, value: Any) -> Dict[str, Any]:
+        simulation.task.set_parameter(param, value)
+        return {param: value}
+
+    setA = partial(param_update, param="a")
+    setB = partial(param_update, param='b')
+    builder.add_sweep_definition(setA, range(2))
+    builder.add_sweep_definition(setB, range(3))
+
+    ts.add_builder(builder)
+
+    experiment = Experiment.from_template(ts)
+    experiment.tags['id'] = experiment.id
+
+    experiment.simulations = list(experiment.simulations)
+    for sim in list(experiment.simulations):
+        sim.tags['id'] = sim.id
+
+    experiment.run(True)
+    sys.exit(0 if experiment.succeeded else -1)

docs/idmtools.ini

@@ -16,6 +16,10 @@ max_local_sims = 6
 # Maximum number of workers processing in parallel
 max_workers = 16
 
+# What type of ids should idmtools use internally
+# use idmtools info plugins id_generators
+id_generator = uuid
+
 # You can also set number of workers per CPU
 # If you had 16 cpus and set to 2, 32 workers would be created
 # workers_per_cpu = 2

docs/plugin_documentation/id-generator-plugins.rst

@@ -0,0 +1,64 @@
+=====================
+ID Generation Plugins
+=====================
+
+**1. Create a file to host the plugin callback for generator (under idmtools_core/idmtools/plugins). The plugin must have the following format**::
+
+    from idmtools.core.interfaces.ientity import IEntity
+
+    from idmtools.registry.hook_specs import function_hook_impl
+
+    @function_hook_impl
+
+    def idmtools_generate_id(item: IEntity) -> str:
+        Args:
+            item: Item for which ID is being generated
+        Returns:
+        return <your id implementation here>
+
+
+The key things in this file are::
+
+    @function_hook_impl
+    def idmtools_generate_id(item: 'IEntity') -> str:
+
+This registers the plugin type with idmtools. By using the name idmtools_generate_id, we know you are defining a callback for ids.
+The callback must match the expected signature.
+
+
+**2. Modify setup.py 'idmtools_hooks' to include the new id generation plugin**::
+
+    entry_points=dict(
+        idmtools_hooks=[
+            "idmtools_id_generate_<name> = <path to plugin>"
+        ]
+    ),
+
+The *label* of the id plugin must start with **idmtools_id_generate_**
+The letters after **idmtools_id_generate_** will be used to select generator in the config.
+
+**3. Modify .ini config file to specify the desired id generator.**
+
+In the .ini configuration file under the 'COMMON' section, use the 'id_generator' option to specify the desired id plugin.
+
+For example, if you want to use the uuid generation plugin ('idmtools_id_generate_uuid'), in the .ini file, you would set the following::
+
+    [COMMON]
+    id_generator = uuid
+
+Similarly, if you want to use the item_sequence plugin ('idmtools_id_generate_item_sequence'), you would specify the following in the .ini file::
+
+    [COMMON]
+    id_generator = item_sequence
+
+The item_sequence plugin allows you to use sequential ids for items in your experiment (experiments themselves as well as simulations, etc).
+You can customize use of this plugin by defining an 'item_sequence' section in the .ini file and using the variables:
+
+    * *sequence_file*: Json file that is used to store the last-used numbers for item ids. For example, if we have one experiment that was defined with two simulations, this file would keep track of the most recently used ids with the following: {"Simulation": 2, "Experiment": 1}. The default for this value is item_sequences.json, and this file will be created in the user's home directory. To note: the sequences start at 0. Also, to note: if an item is generated that does not have the item_type attribute (i.e. Platform), its sequence will be stored under the 'Unknown' key in this json file.
+    * *id_format_str*: This defines the desired format of the item ids (using the sequential id numbers stored in the sequence_file). In this string, one may access the sequential ids by using 'data[item_name]' (which would resolve to the next id #) as well as the 'item_name' (i.e. 'Simulation', 'Experiment'). The default for this value is '{item_name}{data[item_name]:07d}' (which would yield ids of 'Simulation0000000', 'Simulation0000001', etc).
+
+Configuration format::
+
+    [item_sequence]
+    sequence_file = <custom file name>.json
+    id_format_str = '<custom string format>'

docs/plugin_documentation/index.rst

@@ -0,0 +1,9 @@
+=========================
+Plugin Documentation
+=========================
+
+.. toctree::
+   :maxdepth: 3
+   :titlesonly:
+
+   id-generator-plugins

docs/reference.rst

@@ -13,6 +13,20 @@ Packages overview
 .. uml:: /diagrams/packages_overview.puml
 
 
+API and Plugin Documentation
+============================
+.. toctree::
+   :maxdepth: 2
+   :titlesonly:
+
+
+
+   idmtools_index
+   idmtools_models_index
+   idmtools_platform_comps_index
+   idmtools_platform_local_index
+   plugin_documentation/index
+
 Packages and APIs
 =================
 
@@ -60,13 +74,3 @@ API class specifications
 
 .. uml:: /diagrams/apis-emod.puml
 
-.. toctree::
-   :maxdepth: 2
-   :titlesonly:
-
-
-
-   idmtools_index
-   idmtools_models_index
-   idmtools_platform_comps_index
-   idmtools_platform_local_index

idmtools_core/idmtools/core/interfaces/iitem.py

@@ -4,10 +4,14 @@
 Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
 """
 from dataclasses import dataclass, field, fields
+from functools import cache
 from inspect import signature
 from logging import getLogger, DEBUG
 from typing import List, Callable, TYPE_CHECKING, Any, Dict
-from uuid import UUID, uuid4
+from uuid import UUID
+
+from idmtools import IdmConfigParser
+from idmtools.registry.functions import FunctionPluginManager
 from idmtools.utils.hashing import ignore_fields_in_dataclass_on_pickle
 
 if TYPE_CHECKING:  # pragma: no cover
@@ -18,6 +22,21 @@
 PRE_POST_CREATION_HOOK = Callable[['IItem', 'IPlatform'], None]
 
 
+@cache
+def get_id_generator():
+    """
+    Retrieves the type of id generator specified in .ini config as well as corresponding plugin.
+
+    Returns:
+        id_gen: specified id generation plugin in .ini config (uuid, item_sequence, etc)
+        plugin: id generation plugin that is used to determine ids for items. See setup.py > entry_points > idmtools_hooks for full names of plugin options
+    """
+    fpm = FunctionPluginManager.instance()
+    id_gen = IdmConfigParser.get_option(None, "id_generator", "uuid")
+    plugin = fpm.get_plugin(f"idmtools_id_generate_{id_gen}")
+    return id_gen, plugin
+
+
 @dataclass(repr=False)
 class IItem:
     """
@@ -41,7 +60,14 @@ def uid(self):
             ID
         """
         if self._uid is None:
-            self._uid = str(uuid4())
+            id_gen, plugin = get_id_generator()
+            if plugin:
+                self._uid = plugin.idmtools_generate_id(self)
+            else:
+                fpm = FunctionPluginManager.instance()
+                id_plugins = sorted([x[0] for x in fpm.list_name_plugin() if x[0].startswith("idmtools_id_generate_")])
+                raise RuntimeError(f"Could not find the id plugin idmtools_id_generate_{id_gen} defined by id_generator in your idmtools.ini."
+                                   f"Please use one of the following plugins: {', '.join(id_plugins)}")
         return self._uid
 
     @uid.setter