Ianhelle/bokeh3.7 fixes 2025 03 31 (#843)

* Compat fixes for Bokeh 3.7
Also suppressing classes of mypy errors in bokeh modules since it is broadly incompatible with Bokeh.
Fixing flake8 error in common/utilty/types

* Fix for vt-py changing structure VTObject dictionary - causing conversion to dataframe to contain hundreds of columns.
Limiting json_normalize to max_levels=0

* Fix for using AzCli credential with Managed Identity

* Mypy fix for azure_auth_core.py

* Fixing logic in build_cli_client

* Dependency vulns
Updating cryptography to >=43.0.1
Add explicit dependencies for jinja2>=3.1.5 and tornado>=6.4.2 to avoid vulnerable versions

* Update azure-piplelines to skip pip-authenticate

Author: ianhelle

.azurepipelines/azure-pipelines-publish-pypi.yml

@@ -57,21 +57,10 @@ extends:
             versionSpec: '>=3.10'
             architecture: 'x64'
           displayName: 'Use latest Python'
-        - powershell: |
-            # Define the content for pip.conf
-            # Define the content for pip.conf
-            $pipConfContent = "[global]`nindex-url=https://pkgs.dev.azure.com/mstic-detections/mstic-jupyter/_packaging/PyPIInternalFeed/pypi/simple/`nextra-index-url=https://pypi.org/simple"
-            # Create the .pip directory if it doesn't exist
-            New-Item -ItemType Directory -Force -Path "$HOME\.pip"
-            # Write the content to pip.conf
-            $pipConfContent | Out-File -FilePath "$HOME\.pip\pip.conf" -Encoding UTF8
-            # Verify the creation
-            Get-Content "$HOME\.pip\pip.conf"
-          displayName: Create pip.conf
-        - task: PipAuthenticate@1
-          inputs:
-            artifactFeeds: PyPIInternalFeed
-          displayName: 'Authenticate pip for internal feed'
+        #- task: PipAuthenticate@1
+        #  inputs:
+        #    artifactFeeds: PyPIInternalFeed
+        #  displayName: 'Authenticate pip for internal feed'
 
         - powershell: python -m pip install --upgrade pip
           displayName: 'Upgrade pip'

conda/conda-reqs.txt

@@ -14,7 +14,7 @@ azure-storage-blob>=12.5.0
 azure-mgmt-subscription
 beautifulsoup4>=4.0.0
 bokeh>=3.0.0
-cryptography>=3.1
+cryptography>=43.0.1
 deprecated>=1.2.4
 dnspython>=2.0.0, <3.0.0
 folium>=0.9.0
@@ -23,6 +23,7 @@ html5lib
 httpx>=0.23.0, <1.0.0
 ipython>=7.23.1
 ipywidgets>=7.4.2, <9.0.0
+jinja2>=3.1.5  # (sec vuln) transitive dependency via multiple packages
 keyring>=13.2.1
 lxml>=4.6.5
 matplotlib>=3.0.0
@@ -45,6 +46,7 @@ scipy
 setuptools>=40.6.3
 statsmodels
 tldextract>=2.2.2
+tornado>=6.4.2  # (sec vuln) transitive dependency via bokeh
 tqdm>=4.36.1
 typing-extensions>=4.2.0
 urllib3>=1.23

msticpy/auth/azure_auth_core.py

@@ -162,7 +162,7 @@ def _build_cli_client(
         except ClientAuthenticationError as ex:
             logger.info("Azure CLI credential failed to authenticate: %s", str(ex))
             # Check if the error is related to tenant ID
-            if "Tenant" not in str(ex).lower():
+            if "tenant" not in str(ex).casefold():
                 raise  # re-raise if it's a different error
     logger.info("Creating Azure CLI credential without tenant_id")
     cred = AzureCliCredential()

requirements-all.txt

@@ -16,7 +16,7 @@ azure-monitor-query>=1.0.0, <=2.0.0
 azure-storage-blob>=12.5.0
 beautifulsoup4>=4.0.0
 bokeh>=3.0.0
-cryptography>=3.1
+cryptography>=43.0.1
 deprecated>=1.2.4
 dnspython>=2.0.0, <3.0.0
 folium>=0.9.0
@@ -27,6 +27,7 @@ importlib-resources >= 6.4.0; python_version <= "3.8"
 ipython >= 7.1.1; python_version < "3.8"
 ipython >= 7.23.1; python_version >= "3.8"
 ipywidgets>=7.4.2, <9.0.0
+jinja2>=3.1.5  # (sec vuln) transitive dependency via multiple packages
 keyring>=13.2.1
 KqlmagicCustom[jupyter-extended]>=0.1.114.post22
 lxml>=4.6.5
@@ -58,6 +59,7 @@ splunk-sdk>=1.6.0,!=2.0.0
 statsmodels>=0.11.1
 sumologic-sdk>=0.1.11
 tldextract>=2.2.2
+tornado>=6.4.2  # (sec vuln) transitive dependency via bokeh
 tqdm>=4.36.1
 typing-extensions>=4.2.0
 urllib3>=1.23

requirements.txt

@@ -9,7 +9,7 @@ azure-mgmt-subscription>=3.0.0
 azure-monitor-query>=1.0.0, <=2.0.0
 beautifulsoup4>=4.0.0
 bokeh>=3.0.0
-cryptography>=3.1
+cryptography>=43.0.1
 deprecated>=1.2.4
 dnspython>=2.0.0, <3.0.0
 folium>=0.9.0
@@ -20,6 +20,7 @@ importlib-resources >= 6.4.0; python_version <= "3.8"
 ipython >= 7.1.1; python_version < "3.8"
 ipython >= 7.23.1; python_version >= "3.8"
 ipywidgets>=7.4.2, <9.0.0
+jinja2>=3.1.5  # (sec vuln) transitive dependency via multiple packages
 keyring>=13.2.1
 lxml>=4.6.5
 msal>=1.12.0
@@ -40,6 +41,7 @@ pytz>=2019.2  # pandas
 pyyaml>=3.13
 setuptools>=40.6.3
 tldextract>=2.2.2
+tornado>=6.4.2  # (sec vuln) transitive dependency via bokeh
 tqdm>=4.36.1
 typing-extensions>=4.2.0
 urllib3>=1.23

tests/auth/test_azure_auth_core.py

@@ -14,7 +14,11 @@
 
 import pytest
 import pytest_check as check
-from azure.identity import ChainedTokenCredential, DeviceCodeCredential
+from azure.identity import (
+    AzureCliCredential,
+    ChainedTokenCredential,
+    DeviceCodeCredential,
+)
 
 from msticpy.auth.azure_auth_core import (
     AzCredentials,
@@ -305,14 +309,6 @@ def test_build_env_client_alt(
         assert result is None
 
 
-@patch("msticpy.auth.azure_auth_core.AzureCliCredential", autospec=True)
-def test_build_cli_client(mock_cli_credential):
-    """Test _build_cli_client function."""
-    _build_cli_client()
-    # assert isinstance(result, mock_cli_credential)
-    mock_cli_credential.assert_called_once()
-
-
 @pytest.mark.parametrize(
     "env_vars, tenant_id, aad_uri, client_id",
     [
@@ -812,8 +808,94 @@ def side_effect_func(
             1 for instance in mock_instances if instance.get_token.called
         )
         assert token_calls_made > 0
-    # This assertion was already done above when checking mock_instances, so no need to do it twice
+    # This assertion was already done above when checking mock_instances,
+    # so no need to do it twice
     # with different instances that may not have been called properly
     elif side_effect is None:
         # Simple case: verify the token was requested at least once
         assert mock_msi_credential.return_value.get_token.called
+
+
+@pytest.mark.parametrize(
+    "tenant_id, token_side_effects, expected_exception, expected_logs",
+    [
+        # Case 1: tenant_id provided, get_token succeeds
+        (
+            "some-tenant-id",
+            [None],  # No exception from get_token
+            None,
+            ["Creating Azure CLI credential with tenant_id"],
+        ),
+        # Case 2: tenant_id provided, get_token fails with a non-tenant error
+        (
+            "some-tenant-id",
+            [ClientAuthenticationError("Some other error")],
+            ClientAuthenticationError,
+            [
+                "Creating Azure CLI credential with tenant_id",
+                "Azure CLI credential failed to authenticate: Some other error",
+            ],
+        ),
+        # Case 3: tenant_id provided, get_token fails due to tenant error
+        (
+            "some-tenant-id",
+            [ClientAuthenticationError("tenant error")],
+            None,
+            [
+                "Creating Azure CLI credential with tenant_id",
+                "Azure CLI credential failed to authenticate: tenant error",
+                "Creating Azure CLI credential without tenant_id",
+            ],
+        ),
+        # Case 4: no tenant_id, get_token succeeds
+        (
+            None,
+            [None],
+            None,
+            ["Creating Azure CLI credential without tenant_id"],
+        ),
+        # Case 5: no tenant_id, get_token fails
+        (
+            None,
+            [ClientAuthenticationError("No tenant set error")],
+            ClientAuthenticationError,
+            ["Creating Azure CLI credential without tenant_id"],
+        ),
+    ],
+)
+def test_build_cli_client(
+    tenant_id,
+    token_side_effects,
+    expected_exception,
+    expected_logs,
+    caplog,
+    monkeypatch,
+):
+    """Test _build_cli_client with various tenant_id cases and token side effects."""
+    # Set caplog level to capture INFO messages
+    caplog.set_level(logging.INFO)
+
+    mock_cred = MagicMock(spec=AzureCliCredential)
+    # Configure the mock to apply side effect only on first call
+    if token_side_effects:
+        mock_cred.get_token.side_effect = token_side_effects + [
+            None
+        ]  # First call uses side effect, then return None
+
+    # Patch the AzureCliCredential constructor to return the mock
+    with patch(
+        "msticpy.auth.azure_auth_core.AzureCliCredential", return_value=mock_cred
+    ):
+        if expected_exception:
+            with pytest.raises(expected_exception):
+                _build_cli_client(tenant_id=tenant_id)
+        else:
+            returned_cred = _build_cli_client(tenant_id=tenant_id)
+            # We should get our mock credential or a new one if it tries fallback
+            assert isinstance(returned_cred, AzureCliCredential)
+
+    found_logs = [rec.message for rec in caplog.records if rec.levelname == "INFO"]
+    for log_msg in expected_logs:
+        assert any(
+            log_msg in msg for msg in found_logs
+        ), f"Expected log message not found: {log_msg}"