Fix errors in the tests (blue-yonder#457)

* fix bug where predict needs 2D array

* deprecate check_fs_sig_bh method

* disable tempfile auto-delete to work on WinNT

* fix tempdir delete under WinNT

notebooks/timeseries_forecasting_basic_example.ipynb

@@ -625,7 +625,7 @@
     "\n",
     "for i in range(1, len(y)):\n",
     "    ada.fit(X.iloc[:i], y[:i])\n",
-    "    y_pred[i] = ada.predict(X.iloc[i, :])\n",
+    "    y_pred[i] = ada.predict(X.iloc[i, :].values.reshape((1, -1)))\n",
     "    \n",
     "y_pred = pd.Series(data=y_pred, index=y.index)"
    ]

notebooks/visualize-benjamini-yekutieli-procedure.ipynb

@@ -42,7 +42,7 @@
     "\n",
     "from tsfresh.examples.robot_execution_failures import download_robot_execution_failures, load_robot_execution_failures\n",
     "from tsfresh import defaults, extract_features\n",
-    "from tsfresh.feature_selection.feature_selector import check_fs_sig_bh\n",
+    "from tsfresh.feature_selection.relevance import calculate_relevance_table\n",
     "from tsfresh.utilities.dataframe_functions import impute\n",
     "from tsfresh.feature_extraction import ComprehensiveFCParameters\n",
     "\n",
@@ -325,7 +325,7 @@
     }
    ],
    "source": [
-    "df_pvalues_mann = check_fs_sig_bh(X, y, fdr_level=FDR_LEVEL, test_for_binary_target_real_feature='mann')"
+    "df_pvalues_mann = calculate_relevance_table(X, y, fdr_level=FDR_LEVEL, test_for_binary_target_real_feature='mann')"
    ]
   },
   {
@@ -349,8 +349,8 @@
    ],
    "source": [
     "print(\"# total \\t\", len(df_pvalues_mann))\n",
-    "print(\"# relevant \\t\", (df_pvalues_mann[\"rejected\"] == True).sum())\n",
-    "print(\"# irrelevant \\t\", (df_pvalues_mann[\"rejected\"] == False).sum(),\n",
+    "print(\"# relevant \\t\", (df_pvalues_mann[\"relevant\"] == True).sum())\n",
+    "print(\"# irrelevant \\t\", (df_pvalues_mann[\"relevant\"] == False).sum(),\n",
     "      \"( # constant\", (df_pvalues_mann[\"type\"] == \"const\").sum(), \")\")"
    ]
   },
@@ -509,7 +509,7 @@
     }
    ],
    "source": [
-    "df_pvalues_smir = check_fs_sig_bh(X, y, fdr_level=FDR_LEVEL, test_for_binary_target_real_feature='smir')"
+    "df_pvalues_smir = calculate_relevance_table(X, y, fdr_level=FDR_LEVEL, test_for_binary_target_real_feature='smir')"
    ]
   },
   {
@@ -533,8 +533,8 @@
    ],
    "source": [
     "print(\"# total \\t\", len(df_pvalues_smir))\n",
-    "print(\"# relevant \\t\", (df_pvalues_smir[\"rejected\"] == True).sum())\n",
-    "print(\"# irrelevant \\t\", (df_pvalues_smir[\"rejected\"] == False).sum(),\n",
+    "print(\"# relevant \\t\", (df_pvalues_smir[\"relevant\"] == True).sum())\n",
+    "print(\"# irrelevant \\t\", (df_pvalues_smir[\"relevant\"] == False).sum(),\n",
     "      \"( # constant\", (df_pvalues_smir[\"type\"] == \"const\").sum(), \")\")"
    ]
   },
@@ -770,10 +770,10 @@
    "source": [
     "df_pvalues_mann.index = pd.Series(range(0, len(df_pvalues_mann.index)))\n",
     "\n",
-    "df_pvalues_mann.p_value.where(df_pvalues_mann.rejected)\\\n",
+    "df_pvalues_mann.p_value.where(df_pvalues_mann.relevant)\\\n",
     "    .plot(style=\".\", label=\"relevant features\")\n",
     "\n",
-    "df_pvalues_mann.p_value.where(~df_pvalues_mann.rejected & (df_pvalues_mann.type != \"const\"))\\\n",
+    "df_pvalues_mann.p_value.where(~df_pvalues_mann.relevant & (df_pvalues_mann.type != \"const\"))\\\n",
     "    .plot(style=\".\", label=\"irrelevant features\")\n",
     "\n",
     "df_pvalues_mann.p_value.fillna(1).where(df_pvalues_mann.type == \"const\")\\\n",
@@ -830,10 +830,10 @@
    "source": [
     "df_pvalues_smir.index = pd.Series(range(0, len(df_pvalues_smir.index)))\n",
     "\n",
-    "df_pvalues_smir.p_value.where(df_pvalues_smir.rejected)\\\n",
+    "df_pvalues_smir.p_value.where(df_pvalues_smir.relevant)\\\n",
     "    .plot(style=\".\", label=\"relevant features\")\n",
     "\n",
-    "df_pvalues_smir.p_value.where(~df_pvalues_smir.rejected & (df_pvalues_smir.type != \"const\"))\\\n",
+    "df_pvalues_smir.p_value.where(~df_pvalues_smir.relevant & (df_pvalues_smir.type != \"const\"))\\\n",
     "    .plot(style=\".\", label=\"irrelevant features\")\n",
     "\n",
     "df_pvalues_smir.p_value.fillna(1).where(df_pvalues_smir.type == \"const\")\\\n",
@@ -899,14 +899,14 @@
     }
    ],
    "source": [
-    "last_rejected_index = (df_pvalues_mann[\"rejected\"] == True).sum() - 1\n",
+    "last_rejected_index = (df_pvalues_mann[\"relevant\"] == True).sum() - 1\n",
     "margin = 20\n",
     "a = max(last_rejected_index - margin, 0)\n",
     "b = min(last_rejected_index + margin, len(df_pvalues_mann) - 1)\n",
     "\n",
-    "df_pvalues_mann[a:b].p_value.where(df_pvalues_mann[a:b].rejected)\\\n",
+    "df_pvalues_mann[a:b].p_value.where(df_pvalues_mann[a:b].relevant)\\\n",
     "    .plot(style=\".\", label=\"relevant features\")\n",
-    "df_pvalues_mann[a:b].p_value.where(~df_pvalues_mann[a:b].rejected)\\\n",
+    "df_pvalues_mann[a:b].p_value.where(~df_pvalues_mann[a:b].relevant)\\\n",
     "    .plot(style=\".\", label=\"irrelevant features\")\n",
     "plt.plot(np.arange(a, b), rejection_line_mann[a:b], label=\"rejection line (FDR = \" + str(FDR_LEVEL) + \")\")\n",
     "plt.xlabel(\"Feature #\")\n",
@@ -957,14 +957,14 @@
     }
    ],
    "source": [
-    "last_rejected_index = (df_pvalues_smir[\"rejected\"] == True).sum() - 1\n",
+    "last_rejected_index = (df_pvalues_smir[\"relevant\"] == True).sum() - 1\n",
     "margin = 20\n",
     "a = max(last_rejected_index - margin, 0)\n",
     "b = min(last_rejected_index + margin, len(df_pvalues_smir) - 1)\n",
     "\n",
-    "df_pvalues_smir[a:b].p_value.where(df_pvalues_smir[a:b].rejected)\\\n",
+    "df_pvalues_smir[a:b].p_value.where(df_pvalues_smir[a:b].relevant)\\\n",
     "    .plot(style=\".\", label=\"relevant features\")\n",
-    "df_pvalues_smir[a:b].p_value.where(~df_pvalues_smir[a:b].rejected)\\\n",
+    "df_pvalues_smir[a:b].p_value.where(~df_pvalues_smir[a:b].relevant)\\\n",
     "    .plot(style=\".\", label=\"irrelevant features\")\n",
     "plt.plot(np.arange(a, b), rejection_line_smir[a:b], label=\"rejection line (FDR = \" + str(FDR_LEVEL) + \")\")\n",
     "plt.xlabel(\"Feature #\")\n",

tests/integrations/test_notebooks.py

@@ -32,7 +32,12 @@ def _notebook_run(path, timeout=default_timeout):
     except:
         pass
 
-    with tempfile.NamedTemporaryFile(mode='w+t', suffix=".ipynb") as fout:
+    # Ensure temporary files are not auto-deleted as processes have limited 
+    # permissions to re-use file handles under WinNT-based operating systems.
+    fname = ''
+    with tempfile.NamedTemporaryFile(mode='w+t', suffix=".ipynb", delete=False) as fout:
+        fname = fout.name
+
         args = ["jupyter", "nbconvert",
                 "--to", "notebook", "--execute", execproc_timeout]
         if six.PY2:
@@ -44,6 +49,7 @@ def _notebook_run(path, timeout=default_timeout):
 
         fout.seek(0)
         nb = nbformat.read(fout, nbformat.current_nbformat)
+    os.remove(fname)
 
     errors = [output for cell in nb.cells if "outputs" in cell
               for output in cell["outputs"] \

tests/units/scripts/test_run_tsfresh.py

@@ -18,6 +18,7 @@ def setUp(self):
         # Create a temporary directory
         self.test_dir = tempfile.mkdtemp()
         # Change into the tmp dir
+        self.curr_dir = os.getcwd()
         os.chdir(self.test_dir)
 
         def extract_features_mock(df, **kwargs):
@@ -31,6 +32,7 @@ def extract_features_mock(df, **kwargs):
 
     def tearDown(self):
         # Remove the directory after the test
+        os.chdir(self.curr_dir)
         shutil.rmtree(self.test_dir)
         # Disable the mocking
         self.patcher.stop()