new scripts

bench-lp/.ipynb_checkpoints/README-checkpoint.md

@@ -0,0 +1,37 @@
+# Benchmarking ABIP
+
+## Install
+
+See `../install.m` for details
+
+Ensure `ortools`:
+```
+pip install ortools==9.3.10497
+```
+## How to use 
+
+Invoke the following command in bash:
+```bash
+bash run_all_abip.sh
+```
+to see the helper information.
+
+An example run could be: (suppose working directory is at `ABIP/`)
+```bash
+bash lpbench/run_all_abip.sh \
+    dir_in \
+    dir_out \
+    1000 \
+    4 \
+    0
+```
+where 
+```
+- dir_in  <a directory containing mps files>
+- dir_out <the output directory to save results>
+- 1000    <the timelimit of each instance>
+- 4       <the precision, 4 means to solve a problem with tolerance of 0.0001>
+- 0       <0-use direct method, 1-use PCG to solve linear systems>
+```
+
+Once you run this script, you will see a file `cmd.sh` in this directory. Then you can run the command one-by-one, or use `xargs` to run in parallel.

bench-lp/.ipynb_checkpoints/analyze-checkpoint.py

@@ -0,0 +1,140 @@
+# test script to summarize all results
+# @author Chuwen Zhang
+# @usage: python analyze.py conf.json
+#
+import json
+import logging
+import os
+import time
+
+import pandas as pd
+
+from analyze_google_pdhg import google_pdhg_string_to_result
+from analyze_abip import abip_string_to_result
+# from analyze_gurobi import gurobi_string_to_result
+# from analyze_scs import scs_string_to_result
+# from analyze_copt import copt_string_to_result
+
+DEFAULT_CONF = "./conf.analyze.json"
+ANALYZE_METHOD_REGISTRY = {
+    # "gurobi_simplex": gurobi_string_to_result,
+    # "gurobi_dual": gurobi_string_to_result,
+    # "gurobi_barrier": gurobi_string_to_result,
+    "google_pdhg_1e-6": google_pdhg_string_to_result,
+    "google_pdhg_1e-4": google_pdhg_string_to_result,
+    "google_pdhg_1e-8": google_pdhg_string_to_result,
+    # "scs_indirect_1e-4": scs_string_to_result,
+    # "scs_indirect_1e-6": scs_string_to_result,
+    # "scs_direct_1e-4": scs_string_to_result,
+    # "scs_direct_1e-6": scs_string_to_result,
+    "abip_direct_1e-4": abip_string_to_result,
+    "abip_indirect_1e-4": abip_string_to_result,
+    "abip_direct_1e-6": abip_string_to_result,
+    "abip_indirect_1e-6": abip_string_to_result,
+    # "copt_barrier": copt_string_to_result,
+    ###########################
+    "pdlp_julia": google_pdhg_string_to_result,
+    "pdlp": google_pdhg_string_to_result,
+    "abip": abip_string_to_result
+    ###########################
+    # precision not achievable
+    ###########################
+    # "scs_indirect_1e-8": scs_string_to_result,
+    # "scs_direct_1e-8": scs_string_to_result,
+}
+
+FORMAT = '%(asctime)s %(levelname)s %(name)s %(message)s'
+logging.basicConfig(format=FORMAT, level=logging.DEBUG)
+
+logger = logging.getLogger("@analyze")
+logger.setLevel(logging.DEBUG)
+
+
+def analyze(fpath=DEFAULT_CONF):
+    config = json.load(open(fpath, 'r'))
+    instance_path = config['directory']
+    dfs = []
+    logger.info(f"registered methods {config['methods']}")
+    for m in config['methods']:
+        name = m['name']
+        affix = m['affix']
+        funcname = m.get('funcname', "")
+        func = ANALYZE_METHOD_REGISTRY[name] if name in ANALYZE_METHOD_REGISTRY else ANALYZE_METHOD_REGISTRY[funcname]
+        solution_path = os.path.join(instance_path, m['solution_dir'])
+        results = []
+        logger.debug(f"try {m}")
+        if not os.path.exists(solution_path):
+            logger.debug(f"method {m} does not exist")
+            continue
+        logger.debug(f"analyze {m} @ {solution_path}")
+        for _fp in os.listdir(solution_path):
+            fp = os.path.join(solution_path, _fp)
+            if not fp.endswith(affix):
+                continue
+            try:
+                r = func(fp)
+                results.append(r)
+            except Exception as e:
+                logging.exception(e)
+                logging.error(f"failed to analyze {name} @ {fp}")
+        if len(results) > 0:
+            df = pd.DataFrame.from_records(results).assign(
+                method=name,
+                name=lambda df: df['name'].apply(
+                    lambda x: x.replace('.mps', '').replace('.gz', '').replace('.mat', '')
+                )
+            ).drop_duplicates(subset=['name', 'method'])
+            dfs.append(df)
+
+        logger.info(f"{name} solution finished")
+    df_agg = pd.concat(dfs).fillna(0)
+
+    instances = df_agg['name'].unique()
+    method_names = [n['name'] for n in config['methods']]
+    index = pd.MultiIndex.from_tuples(
+        list((i, m) for m in method_names for i in instances.tolist()),
+        names=('name', 'method')
+    )
+    df_agg = df_agg.set_index(
+        ['name', 'method']
+    ).reindex(
+        index, fill_value='-'
+    ).reset_index(drop=False).sort_values(
+        ['name', 'method']
+    ).assign(
+        real_name=lambda df: df['name'].apply(lambda x: x.replace('pre_', ''))
+    )
+
+    return df_agg
+
+
+if __name__ == '__main__':
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--conf", type=str)
+    # only output the instances with prefix
+    parser.add_argument("--prefix", type=str, default='pre')
+
+    args = parser.parse_args()
+    df_agg = analyze(args.conf)
+    result_stamp = int(time.time())
+    df_agg.to_excel(f"result_{result_stamp}.xlsx", index=False)
+    idx_df_agg_with_prefix = df_agg['name'].apply(lambda x: x.startswith(args.prefix))
+    df_agg_pre = df_agg[idx_df_agg_with_prefix]
+    # is successful? todo, may have to add a ground truth
+    bool_fail = lambda x: x.lower() in ['unfinished']
+    df_agg_pre = df_agg_pre.assign(
+        bool_fail=lambda df: df.sol_status.apply(bool_fail)
+    )
+    if args.prefix.__len__() > 0:
+        df_agg_pre.to_excel(f"result_{result_stamp}.{args.prefix}.xlsx", index=False)
+    # do some summary stats
+    logger.info(f"analyze finished to with stamp {result_stamp}")
+
+    df_agg_suc = df_agg_pre[~df_agg_pre.bool_fail]
+    df_sum = df_agg_suc[(df_agg_suc != '-')].dropna().groupby(
+        'method'
+    )['name'].count().rename('solved').reset_index()
+    df_sum.to_excel(f"result_{result_stamp}.summary.xlsx", index=False)
+    print(df_sum.to_latex(multirow=True, multicolumn=True, caption='Number of solved instances'))

bench-lp/.ipynb_checkpoints/analyze_abip-checkpoint.py

@@ -0,0 +1,27 @@
+"""typical abip schema
+{
+}
+"""
+import json
+import pandas as pd
+
+
+def abip_string_to_result(fpath):
+  with open(fpath, 'r') as f:
+    content = json.load(f)
+    res_primal = content['pres']
+    res_dual = content['dres']
+    sol_time = content['time']
+    sol_status = content['status']
+    val_primal = content['pobj']
+    val_dual = content['dobj']
+    name = fpath.split("/")[-1].split(".")[0]
+    return dict(iteration_num=content['admm_iter'],
+                ipm_num=content['ipm_iter'],
+                res_primal=res_primal,
+                res_dual=res_dual,
+                sol_time=sol_time,
+                val_primal=val_primal,
+                val_dual=val_dual,
+                sol_status=sol_status,
+                name=name)

bench-lp/.ipynb_checkpoints/analyze_crs-checkpoint.py

@@ -0,0 +1,159 @@
+# test script to summarize all results
+# used for collect crossover results.
+# @author Chuwen Zhang
+#
+import json
+import logging
+import os
+import time
+
+import pandas as pd
+
+DEFAULT_CONF = "./conf.analyze.json"
+
+FORMAT = "%(asctime)s %(levelname)s %(name)s %(message)s"
+logging.basicConfig(format=FORMAT, level=logging.DEBUG)
+
+logger = logging.getLogger("@analyze")
+logger.setLevel(logging.DEBUG)
+
+
+def copt_crossover_to_dict(fpath):
+    """
+    {
+      "name": "pre_25fv47",
+      "Status": 1,
+      "ObjVal": 5501.845888286746,
+      "IterCount": 2802,
+      "SolvingTime": 0.1771249771118164
+    }
+    """
+    with open(fpath, "r") as f:
+        content = json.load(f)
+        iternum = content["IterCount"]
+        soltime = content["SolvingTime"]
+        status = content["Status"]
+        objval = content["ObjVal"]
+        name = content["name"]
+        return dict(
+            name=name, iteration=iternum, time=soltime, status=status, objval=objval,
+        )
+
+
+def analyze(fpath=DEFAULT_CONF):
+    config = json.load(open(fpath, "r"))
+    instance_path = config["directory"]
+    dfs = []
+    logger.info(f"registered methods {config['methods']}")
+    for m in config["methods"]:
+        name = m["name"]
+        affix = m["affix"]
+        solution_path = os.path.join(instance_path, m["solution_dir"])
+        results = []
+        logger.debug(f"try {m}")
+        if not os.path.exists(solution_path):
+            logger.debug(f"method {m} does not exist")
+            continue
+        logger.debug(f"analyze {m} @ {solution_path}")
+        for _fp in os.listdir(solution_path):
+            fp = os.path.join(solution_path, _fp)
+            if not fp.endswith(affix):
+                continue
+            try:
+                r = copt_crossover_to_dict(fp)
+                results.append(r)
+            except Exception as e:
+                logging.exception(e)
+                logging.error(f"failed to analyze {name} @ {fp}")
+        if len(results) > 0:
+            df = (
+                pd.DataFrame.from_records(results)
+                .assign(
+                    method=name,
+                    name=lambda df: df["name"].apply(
+                        lambda x: x.replace(".mps", "").replace(".gz", "")
+                    ),
+                )
+                .drop_duplicates(subset=["name", "method"])
+            )
+            dfs.append(df)
+
+        logger.info(f"{name} solution finished")
+    df_agg = pd.concat(dfs).fillna(0)
+
+    instances = df_agg["name"].unique()
+    method_names = [n["name"] for n in config["methods"]]
+    index = pd.MultiIndex.from_tuples(
+        list((i, m) for m in method_names for i in instances.tolist()),
+        names=("name", "method"),
+    )
+    df_agg = (
+        df_agg.set_index(["name", "method"])
+        .reindex(index, fill_value="-")
+        .reset_index(drop=False)
+        .sort_values(["name", "method"])
+        .assign(real_name=lambda df: df["name"].apply(lambda x: x.replace("pre_", "")))
+    )
+
+    return df_agg
+
+
+def scaled_gmean(arr, scale=1):
+    """compute scaled geometric mean,
+    the scale=10 mimics the Hans's benchmark
+    """
+    return stats.gmean(arr + scale) - scale
+
+
+if __name__ == "__main__":
+    import argparse
+    from scipy import stats
+
+    pd.set_option("display.max_columns", None)
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--conf", type=str)
+    # only output the instances with prefix
+    parser.add_argument(
+        "--prefix", type=str, default="pre", help="only analyze results with prefix"
+    )
+
+    args = parser.parse_args()
+    df_agg = analyze(args.conf)
+    result_stamp = int(time.time())
+    df_agg.to_excel(f"result_{result_stamp}.xlsx", index=False)
+    idx_df_agg_with_prefix = df_agg["name"].apply(lambda x: x.startswith(args.prefix))
+    df_agg_pre = df_agg[idx_df_agg_with_prefix]
+    # is successful? todo, may have to add a ground truth
+    bool_fail = lambda x: x != 1
+    df_agg_pre = df_agg_pre.assign(bool_fail=lambda df: df.status.apply(bool_fail))
+    df_agg_pre.to_excel(f"result_{result_stamp}.{args.prefix}.xlsx", index=False)
+    # do some summary stats
+    logger.info(f"analyze finished to with stamp {result_stamp}")
+
+    df_agg_suc = df_agg_pre[~df_agg_pre.bool_fail]
+    df_agg_suc = df_agg_suc[(df_agg_suc != "-")].dropna()
+    # df_sum = df_agg_suc.groupby(
+    #     'method'
+    # )['name'].count().rename('solved').reset_index()
+    df_sum = (
+        df_agg_suc.replace(
+            {"time": {"-": 1e3}, "iteration": {"-": 1e4}, "status": {"-": 0}}
+        )
+        .groupby("method")
+        .agg({"status": sum, "iteration": scaled_gmean, "time": scaled_gmean})
+    )
+    df_sum.to_excel(f"result_{result_stamp}.summary.xlsx", index=True)
+    print(
+        r"""
+\documentclass{article}
+\usepackage{lscape,longtable,multirow}
+\usepackage{booktabs,caption}
+\begin{document}
+    """
+        + f"""
+{df_sum.to_latex(multirow=True,longtable=True, multicolumn=True, caption='Number of solved instances')}
+    """
+        + r"""
+\end{document}
+    """
+    )

bench-lp/.ipynb_checkpoints/analyze_google_pdhg-checkpoint.py

@@ -0,0 +1,30 @@
+import json
+import pandas as pd
+
+
+def google_pdhg_string_to_result(fpath):
+  with open(fpath, 'r') as f:
+    content = json.load(f)
+    stats_solution = content['solutionStats']
+    res_primal = stats_solution['convergenceInformation'][0]["lInfPrimalResidual"]
+    res_dual = stats_solution['convergenceInformation'][0]["lInfDualResidual"]
+    sol_time = content['solveTimeSec']
+    sol_status = "optimal" if "optimal" in content['terminationReason'].lower() else "unfinished"
+    val_primal = stats_solution['convergenceInformation'][0]["primalObjective"]
+    val_dual = stats_solution['convergenceInformation'][0]["dualObjective"]
+
+    return dict(iteration_num=content['iterationCount'],
+                res_primal=res_primal.__round__(8),
+                res_dual=res_dual.__round__(8),
+                sol_time=sol_time.__round__(4),
+                val_primal=val_primal.__round__(4),
+                val_dual=val_dual.__round__(4),
+                sol_status=sol_status,
+                name=fpath.split("/")[-1].split(".")[0])
+
+
+if __name__ == '__main__':
+  import sys
+  result = google_pdhg_string_to_result(
+    sys.argv[1])
+  print(result)