Fix/materials myopic

temoa/temoa_model/hybrid_loader.py

@@ -101,6 +101,7 @@ def __init__(self, db_connection: Connection, config: TemoaConfig):
         self.viable_rt: ViableSet | None = None
         self.viable_rpit: ViableSet | None = None
         self.viable_rtt: ViableSet | None = None  # to support scanning LinkedTech
+        self.viable_rtv_eol: ViableSet | None = None  # to support scanning EndOfLifeOutput
         self.efficiency_values: list[tuple] = []
 
         # container for loaded data
@@ -175,6 +176,7 @@ def _build_efficiency_dataset(
             self.viable_rt = filts['rt']
             self.viable_rpit = filts['rpit']
             self.viable_rpto = filts['rpto']
+            self.viable_rtv_eol = filts['rtv_eol']
             self.viable_techs = filts['t']
             self.viable_input_comms = filts['ic']
             self.viable_vintages = filts['v']
@@ -977,10 +979,19 @@ def load_indexed_set(indexed_set: Set, index_value, element, element_validator=N
 
         # EmissionEmbodied
         if self.table_exists('EmissionEmbodied'):
-            raw = cur.execute(
-                'SELECT region, emis_comm, tech, vintage, value '
-                'FROM main.EmissionEmbodied'
-            ).fetchall()
+            if mi:
+                qry = (
+                    'SELECT region, emis_comm, tech, vintage, value FROM main.EmissionEmbodied'
+                    ' WHERE vintage >= ? AND vintage <= ?'
+                )
+                raw = cur.execute(
+                    qry,
+                    (mi.base_year, mi.last_demand_year),
+                ).fetchall()
+            else:
+                raw = cur.execute(
+                    'SELECT region, emis_comm, tech, vintage, value FROM main.EmissionEmbodied'
+                ).fetchall()
             load_element(M.EmissionEmbodied, raw, self.viable_rtv, (0, 2, 3))
 
         # EmissionEndOfLife
@@ -989,21 +1000,31 @@ def load_indexed_set(indexed_set: Set, index_value, element, element_validator=N
                 'SELECT region, emis_comm, tech, vintage, value '
                 'FROM main.EmissionEndOfLife'
             ).fetchall()
-            load_element(M.EmissionEndOfLife, raw, self.viable_rtv, (0, 2, 3))
+            load_element(M.EmissionEndOfLife, raw, self.viable_rtv_eol, (0, 2, 3))
 
         # ConstructionInput
         if self.table_exists('ConstructionInput'):
-            raw = cur.execute(
-                'SELECT region, input_comm, tech, vintage, value FROM main.ConstructionInput'
-            ).fetchall()
+            if mi:
+                qry = (
+                    'SELECT region, input_comm, tech, vintage, value FROM main.ConstructionInput'
+                    ' WHERE vintage >= ? AND vintage <= ?'
+                )
+                raw = cur.execute(
+                    qry,
+                    (mi.base_year, mi.last_demand_year),
+                ).fetchall()
+            else:
+                raw = cur.execute(
+                    'SELECT region, input_comm, tech, vintage, value FROM main.ConstructionInput'
+                ).fetchall()
             load_element(M.ConstructionInput, raw, self.viable_rtv, (0, 2, 3))
 
         # EndOfLifeOutput
         if self.table_exists('EndOfLifeOutput'):
             raw = cur.execute(
                 'SELECT region, tech, vintage, output_comm, value FROM main.EndOfLifeOutput'
             ).fetchall()
-            load_element(M.EndOfLifeOutput, raw, self.viable_rtv, (0, 1, 2))
+            load_element(M.EndOfLifeOutput, raw, self.viable_rtv_eol, (0, 1, 2))
 
         # LinkedTechs
         # Note:  Both of the linked techs must be viable.  As this is non period/vintage

temoa/temoa_model/model_checking/commodity_network_manager.py

@@ -117,6 +117,7 @@ def build_filters(self) -> dict[str, ViableSet]:
         valid_rpit = set()
         valid_rpto = set()
         valid_t = set()
+        valid_rtv_eol = set()
         valid_input_commodities = set()
         valid_output_commodities = set()
         valid_vintages = set()
@@ -128,6 +129,7 @@ def build_filters(self) -> dict[str, ViableSet]:
                 valid_rpit.add((tech.region, p, tech.ic, tech.name))
                 valid_rpto.add((tech.region, p, tech.name, tech.oc))
                 valid_t.add(tech.name)
+                valid_rtv_eol.add((tech.region, tech.ic, tech.vintage))
                 valid_input_commodities.add(tech.ic)
                 valid_output_commodities.add(tech.oc)
                 valid_vintages.add(tech.vintage)
@@ -148,6 +150,9 @@ def build_filters(self) -> dict[str, ViableSet]:
             'v': ViableSet(elements=valid_vintages),
             'ic': ViableSet(elements=valid_input_commodities),
             'oc': ViableSet(elements=valid_output_commodities),
+            'rtv_eol': ViableSet(
+                elements=valid_rtv_eol, exception_loc=0, exception_vals=ViableSet.REGION_REGEXES
+            ),
         }
         return filts
 

temoa/temoa_model/model_checking/network_model_data.py

@@ -183,7 +183,7 @@ def _build_from_db(
     tech_retire = {t[0] for t in raw}
     raw = cur.execute('SELECT DISTINCT region, tech, vintage FROM LifetimeSurvivalCurve').fetchall()
     tech_survival_curve = set(raw)
-    raw = cur.execute('SELECT period FROM TimePeriod').fetchall()
+    raw = cur.execute('SELECT period FROM TimePeriod WHERE flag == "f"').fetchall()
     periods = [p[0] for p in sorted(raw)]
     period_length = {periods[i]: periods[i+1] - periods[i] for i in range(len(periods)-1)}
     periods = periods[:-1]
@@ -233,19 +233,20 @@ def _build_from_db(
             '     AND main.MyopicEfficiency.region = main.LifeTimeTech.region '
             '   JOIN TimePeriod '
             '   ON MyopicEfficiency.vintage = TimePeriod.period '
-            # f'   WHERE main.MyopicEfficiency.vintage <= {myopic_index.last_demand_year}'
+            f'   WHERE main.MyopicEfficiency.vintage <= {myopic_index.last_demand_year}'
         )
     raw = cur.execute(query).fetchall()
     # need to exclude the final year which is a non-demand year and should have no tech data
     # This ensures that the periods in this will match the periods in the hybrid loader.
 
     # filter further if myopic
     if myopic_index:
-        periods = {
+        periods = [
             p for p in periods if myopic_index.base_year <= p <= myopic_index.last_demand_year
-        }
+        ]
     techs = defaultdict(set)
     living_techs = set()  # for screening the linked techs below
+    living_rtv = set()
     # filter out the dead ones...
     for element in raw:
         (r, ic, tech, v, oc, lifetime) = element
@@ -267,42 +268,55 @@ def _build_from_db(
                 else:
                     techs[r, p].add(Tech(r, ic, tech, v, oc))
                     living_techs.add(tech)
+                    living_rtv.add((r, tech, v))
                     if ic in source_comms:
                         source_dict[r, p].add(ic)
                     if oc in waste_comms:
                         waste_dict[r, p].add(oc)
 
-            # End of life output
-            if any((
-                p <= v+lifetime < p + period_length[p], # natural eol this period
-                tech in tech_retire and v < p <= v+lifetime - period_length[p], # allowed early retirement
-                (r, tech, v) in tech_survival_curve and v <= p <= v+lifetime
-            )):
-                try:
-                    raw_eol = cur.execute(
-                        'SELECT region, tech, vintage, output_comm FROM EndOfLifeOutput '
-                        f' WHERE region == "{r}" AND tech == "{tech}" AND vintage == {v}'
-                    ).fetchall()
-
-                    for _r, _tech, _v, _oc in raw_eol:
-                        techs[_r, p].add(Tech(_r, _tech, 'EndOfLife', _v, _oc))
-                        source_dict[_r, p].add(_tech)
-                        res.capacity_commodities.add(_tech)
-                        living_techs.add(_tech)
-                        if _oc in waste_comms:
-                            waste_dict[_r, p].add(_oc)
-                except:
-                    # EndOfLifeOutput table did not exist TODO remove this eventually
-                    pass
+    # End of life output
+    query = (
+        '  SELECT main.EndOfLifeOutput.region, EndOfLifeOutput.tech, EndOfLifeOutput.vintage, EndOfLifeOutput.output_comm,  '
+        f'  coalesce(main.LifetimeProcess.lifetime, main.LifetimeTech.lifetime, {default_lifetime}) AS lifetime '
+        '   FROM main.EndOfLifeOutput '
+        '    LEFT JOIN main.LifetimeProcess '
+        '       ON main.EndOfLifeOutput.tech = LifetimeProcess.tech '
+        '       AND main.EndOfLifeOutput.vintage = LifetimeProcess.vintage '
+        '       AND main.EndOfLifeOutput.region = LifetimeProcess.region '
+        '    LEFT JOIN main.LifetimeTech '
+        '       ON main.EndOfLifeOutput.tech = main.LifetimeTech.tech '
+        '     AND main.EndOfLifeOutput.region = main.LifeTimeTech.region '
+        '   JOIN TimePeriod '
+        '   ON EndOfLifeOutput.vintage = TimePeriod.period '
+    )
+    raw = cur.execute(query).fetchall()
+    for (r, tech, v, oc, lifetime) in raw:
+        for p in periods:
+            if (
+                (p == periods[0] and v + lifetime == p) # retires on start of horizon
+                or (
+                    (r, tech, v) in living_rtv and any((
+                        p <= v+lifetime < p + period_length[p], # natural eol this period
+                        tech in tech_retire and v < p <= v+lifetime - period_length[p], # allowed early retirement
+                        (r, tech, v) in tech_survival_curve and v <= p <= v+lifetime # survival curve retirement
+                    ))
+                )
+            ):
+                techs[r, p].add(Tech(r, tech, tech, v, oc))
+                source_dict[r, p].add(tech)
+                res.capacity_commodities.add(tech)
+                if oc in waste_comms:
+                    waste_dict[r, p].add(oc)
 
     # Construction input
     try:
         raw = cur.execute('SELECT region, input_comm, tech, vintage FROM ConstructionInput').fetchall()
         for r, ic, tech, v in raw:
-            techs[r, v].add(Tech(r, ic, 'Construction', v, tech))
+            if v not in periods:
+                continue
+            techs[r, v].add(Tech(r, ic, tech, v, tech))
             demand_dict[r, v].add(tech)
             res.capacity_commodities.add(tech)
-            living_techs.add(tech)
     except:
         # ConstructionInput table did not exist TODO remove this eventually
         pass

temoa/temoa_model/table_data_puller.py

@@ -99,7 +99,7 @@ def poll_capacity_results(M: TemoaModel, epsilon=1e-5) -> CapData:
     for r, t, v in M.V_NewCapacity.keys():
         if v in M.time_optimize:
             val = value(M.V_NewCapacity[r, t, v])
-            if abs(val) < epsilon:
+            if val < epsilon:
                 continue
             new_cap = (r, t, v, val)
             built.append(new_cap)
@@ -108,7 +108,7 @@ def poll_capacity_results(M: TemoaModel, epsilon=1e-5) -> CapData:
     net = []
     for r, p, t, v in M.V_Capacity.keys():
         val = value(M.V_Capacity[r, p, t, v])
-        if abs(val) < epsilon:
+        if val < epsilon:
             continue
         new_net_cap = (r, p, t, v, val)
         net.append(new_net_cap)
@@ -124,8 +124,8 @@ def poll_capacity_results(M: TemoaModel, epsilon=1e-5) -> CapData:
             if t in M.tech_retirement and v < p <= v + lifetime - value(M.PeriodLength[p]):
                 early = value(M.V_RetiredCapacity[r, p, t, v])
                 eol -= early
-            early = 0 if abs(early) < epsilon else early
-            eol = 0 if abs(eol) < epsilon else eol
+            early = 0 if early < epsilon else early
+            eol = 0 if eol < epsilon else eol
             if early == 0 and eol == 0:
                 continue
             new_retired_cap = (r, p, t, v, eol, early)
@@ -150,7 +150,7 @@ def poll_flow_results(M: TemoaModel, epsilon=1e-5) -> dict[FI, dict[FlowType, fl
     for key in M.V_FlowIn.keys():
         fi = FI(*key)
         flow = value(M.V_FlowIn[fi])
-        if abs(flow) < epsilon:
+        if flow < epsilon:
             continue
         res[fi][FlowType.IN] = flow
         res[fi][FlowType.LOST] = (1 - temoa_rules.get_variable_efficiency(M, *key)) * flow
@@ -159,7 +159,7 @@ def poll_flow_results(M: TemoaModel, epsilon=1e-5) -> dict[FI, dict[FlowType, fl
     for key in M.V_FlowOut.keys():
         fi = FI(*key)
         flow = value(M.V_FlowOut[fi])
-        if abs(flow) < epsilon:
+        if flow < epsilon:
             continue
         res[fi][FlowType.OUT] = flow
 
@@ -172,15 +172,15 @@ def poll_flow_results(M: TemoaModel, epsilon=1e-5) -> dict[FI, dict[FlowType, fl
     for key in M.V_Curtailment.keys():
         fi = FI(*key)
         val = value(M.V_Curtailment[fi])
-        if abs(val) < epsilon:
+        if val < epsilon:
             continue
         res[fi][FlowType.CURTAIL] = val
 
     # flex techs.  This will subtract the flex from their output flow IOT make OUT the "net"
     for key in M.V_Flex.keys():
         fi = FI(*key)
         flow = value(M.V_Flex[fi])
-        if abs(flow) < epsilon:
+        if flow < epsilon:
             continue
         res[fi][FlowType.FLEX] = flow
         res[fi][FlowType.OUT] -= flow
@@ -200,7 +200,7 @@ def poll_flow_results(M: TemoaModel, epsilon=1e-5) -> dict[FI, dict[FlowType, fl
                     distribution = value(M.SegFrac[p, s, d])
                 fi = FI(r, p, s, d, i, t, v, o)
                 flow = value(M.V_FlowOutAnnual[r, p, i, t, v, o]) * distribution
-                if abs(flow) < epsilon:
+                if flow < epsilon:
                     continue
                 res[fi][FlowType.OUT] = flow
                 res[fi][FlowType.IN] = flow / value(M.Efficiency[ritvo(fi)])
@@ -212,7 +212,7 @@ def poll_flow_results(M: TemoaModel, epsilon=1e-5) -> dict[FI, dict[FlowType, fl
             for d in M.time_of_day:
                 fi = FI(r, p, s, d, i, t, v, o)
                 flow = value(M.V_FlexAnnual[r, p, i, t, v, o]) * value(M.SegFrac[p, s, d])
-                if abs(flow) < epsilon:
+                if flow < epsilon:
                     continue
                 res[fi][FlowType.FLEX] = flow
                 res[fi][FlowType.OUT] -= flow
@@ -260,7 +260,7 @@ def poll_storage_level_results(M: TemoaModel, epsilon=1e-5) -> dict[SLI, float]:
             continue
         state = value(M.V_StorageLevel[r, p, s, d, t, v]) / (value(M.SegFracPerSeason[p, s]) * value(M.DaysPerPeriod))
         sli = SLI(r, p, s, d, t, v)
-        if abs(state) < epsilon: state = 0 # still want to know but decimals are ugly
+        if state < epsilon: state = 0 # still want to know but decimals are ugly
         res[sli] = state
 
     for r, p, s_seq, t, v in M.SeasonalStorageLevel_rpstv:
@@ -272,7 +272,7 @@ def poll_storage_level_results(M: TemoaModel, epsilon=1e-5) -> dict[SLI, float]:
         for d in M.time_of_day:
             state = value(M.V_SeasonalStorageLevel[r, p, s_seq, t, v]) + value(M.V_StorageLevel[r, p, s, d, t, v]) * days_adjust
             sli = SLI(r, p, s_seq, d, t, v)
-            if abs(state) < epsilon: state = 0 # still want to know but decimals are ugly
+            if state < epsilon: state = 0 # still want to know but decimals are ugly
             res[sli] = state
 
     return res
@@ -316,7 +316,7 @@ def poll_cost_results(
     for r, t, v in M.CostInvest.sparse_iterkeys():  # Returns only non-zero values
         # gather details...
         cap = value(M.V_NewCapacity[r, t, v])
-        if abs(cap) < epsilon:
+        if cap < epsilon:
             continue
         loan_life = value(LLN[r, t, v])
         loan_rate = value(M.LoanRate[r, t, v])
@@ -374,7 +374,7 @@ def poll_cost_results(
 
     for r, p, t, v in M.CostFixed.sparse_iterkeys():
         cap = value(M.V_Capacity[r, p, t, v])
-        if abs(cap) < epsilon:
+        if cap < epsilon:
             continue
 
         fixed_cost = value(M.CostFixed[r, p, t, v])
@@ -420,7 +420,7 @@ def poll_cost_results(
                 for S_i in M.processInputs[r, p, t, v]
                 for S_o in M.processOutputsByInput[r, p, t, v, S_i]
             )
-        if abs(activity) < epsilon:
+        if activity < epsilon:
             continue
 
         var_cost = value(M.CostVariable[r, p, t, v])