[build] Fix various Jython CI and test issues

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index a1e21fd4aeca2d4f776675c38c9f740020b981ee..6b91edd6c5e2e0a5697c84f241346a1adff49c37 100644 (file)
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -111,10 +111,12 @@ jobs:
           run-tests-ext: bat
         # jython
         - os: ubuntu-20.04
+          python-version: 2.7
           python-impl: jython
           ytdl-test-set: ${{ contains(needs.select.outputs.test-set, 'core') && 'core' || 'nocore' }}
           run-tests-ext: sh
         - os: ubuntu-20.04
+          python-version: 2.7
           python-impl: jython
           ytdl-test-set: ${{ contains(needs.select.outputs.test-set, 'download') && 'download'  || 'nodownload' }}
           run-tests-ext: sh
@@ -163,22 +165,22 @@ jobs:
             'print(sys.path)' \
             | ${expected} -
     #-------- Python 3.12 -
-    - name: Set up Python 3.12 environment
-      if: ${{ matrix.python-version == '3.12' }}
+    - name: Set up CPython 3.12 environment
+      if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == '3.12' }}
       shell: bash
       run: |
         PYENV_ROOT=$HOME/.local/share/pyenv
         echo "PYENV_ROOT=${PYENV_ROOT}" >> "$GITHUB_ENV"
     - name: Cache Python 3.12
       id: cache312
-      if: ${{ matrix.python-version == '3.12' }}
+      if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == '3.12' }}
       uses: actions/cache@v3
       with:
         key: python-3.12
         path: |
           ${{ env.PYENV_ROOT }}
     - name: Build and set up Python 3.12
-      if: ${{ matrix.python-version == '3.12' && ! steps.cache312.outputs.cache-hit }}
+      if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == '3.12' && ! steps.cache312.outputs.cache-hit }}
       # dl and build locally
       shell: bash
       run: |
@@ -192,7 +194,7 @@ jobs:
         git clone "https://github.com/pyenv/pyenv.git" "$PYENV_ROOT"
         pyenv install 3.12.0b4
     - name: Locate Python 3.12
-      if: ${{ matrix.python-version == '3.12' }}
+      if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == '3.12' }}
       shell: bash
       run: |
         PYTHONHOME="${{ env.PYENV_ROOT }}/versions/3.12.0b4"
@@ -200,7 +202,7 @@ jobs:
         echo "PATH=${PYTHONHOME}/bin:$PATH" >> "$GITHUB_ENV"
     #-------- Python 2.7 --
     - name: Set up Python 2.7
-      if: ${{ matrix.python-version == '2.7' }}
+      if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == '2.7' }}
       # install 2.7
       shell: bash
       run: |
@@ -208,7 +210,7 @@ jobs:
         echo "PYTHONHOME=/usr" >> "$GITHUB_ENV"
     #-------- Python 2.6 --
     - name: Set up Python 2.6 environment
-      if: ${{ matrix.python-version == '2.6' }}
+      if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == '2.6' }}
       shell: bash
       run: |
         openssl_name=openssl-1.0.2u
@@ -228,7 +230,7 @@ jobs:
           ${{ env.openssl_dir }}
           ${{ env.PYENV_ROOT }}
     - name: Build and set up Python 2.6
-      if: ${{ matrix.python-version == '2.6' && ! steps.cache26.outputs.cache-hit }}
+      if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == '2.6' && ! steps.cache26.outputs.cache-hit }}
       # dl and build locally
       shell: bash
       run: |
@@ -266,7 +268,7 @@ jobs:
         export LD_LIBRARY_PATH="$openssl_lib"
         pyenv install 2.6.9
     - name: Locate Python 2.6
-      if: ${{ matrix.python-version == '2.6' }}
+      if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == '2.6' }}
       shell: bash
       run: |
         PYTHONHOME="${{ env.PYENV_ROOT }}/versions/2.6.9"
@@ -288,7 +290,7 @@ jobs:
         echo "PIP=pip" >> "$GITHUB_ENV"
     - name: Cache Jython
       id: cachejy
-      if: ${{ matrix.python-impl == 'jython' }}
+      if: ${{ matrix.python-impl == 'jython' && matrix.python-version == '2.7' }}
       uses: actions/cache@v3
       with:
         # 2.7.3 now available, may solve SNI issue
@@ -296,7 +298,7 @@ jobs:
         path: |
           ${{ env.JYTHON_ROOT }}
     - name: Install Jython
-      if: ${{ matrix.python-impl == 'jython' && ! steps.cachejy.outputs.cache-hit }}
+      if: ${{ matrix.python-impl == 'jython' && matrix.python-version == '2.7' && ! steps.cachejy.outputs.cache-hit }}
       shell: bash
       run: |
         JYTHON_ROOT="${{ env.JYTHON_ROOT }}"
@@ -309,6 +311,11 @@ jobs:
       run: |
         JYTHON_ROOT="${{ env.JYTHON_ROOT }}"
         echo "${JYTHON_ROOT}/bin" >> $GITHUB_PATH
+    - name: Install supporting Python 2.7 if possible
+      if: ${{ steps.cachejy.outputs.cache-hit }}
+      shell: bash
+      run: |
+        sudo apt-get install -y python2.7 || true
     #-------- pip ---------
     - name: Set up supported Python ${{ matrix.python-version }} pip
       if: ${{ (matrix.python-version != '3.2' && steps.setup-python.outputs.python-path) || matrix.python-version == '2.7' }}
@@ -391,6 +398,11 @@ jobs:
       if: ${{ contains(needs.select.outputs.test-set, matrix.ytdl-test-set ) }}
       shell: bash
       run: |
+        # set PYTHON_VER
+        PYTHON_VER=${{ matrix.python-version }}
+        [ "${PYTHON_VER#*-}" != "$PYTHON_VER" ] || PYTHON_VER="${{ matrix.python-impl }}-${PYTHON_VER}"
+        echo "PYTHON_VER=$PYTHON_VER" >> "$GITHUB_ENV"
+        echo "PYTHON_IMPL=${{ matrix.python-impl }}" >> "$GITHUB_ENV"
         # define a test to validate the Python version used by nosetests
         printf '%s\n' \
           'from __future__ import unicode_literals' \
@@ -405,7 +417,7 @@ jobs:
           '    def test_python_ver(self):' \
           '        self.assertEqual(["%d" % v for v in sys.version_info[:2]], self.ver[-1].split(".")[:2])' \
           '        self.assertTrue(sys.version.startswith(self.ver[-1]))' \
-          '        self.assertIn(self.ver[0], sys.version.lower())' \
+          '        self.assertIn(self.ver[0], ",".join((sys.version, platform.python_implementation())).lower())' \
           '    def test_python_impl(self):' \
           '        self.assertIn(platform.python_implementation().lower(), (os.environ["PYTHON_IMPL"], self.ver[0]))' \
           > test/test_python.py
@@ -415,8 +427,6 @@ jobs:
       continue-on-error: ${{ matrix.ytdl-test-set == 'download' || matrix.python-impl == 'jython' }}
       env:
         YTDL_TEST_SET: ${{ matrix.ytdl-test-set }}
-        PYTHON_VER: ${{ matrix.python-version }}
-        PYTHON_IMPL: ${{ matrix.python-impl }}
       run: |
         ./devscripts/run_tests.${{ matrix.run-tests-ext }}
   flake8:

diff --git a/devscripts/make_lazy_extractors.py b/devscripts/make_lazy_extractors.py
index 1a841a08b698d19af69d5d1eacd4c4a23e94f94c..dee9d6d912a09922b33793bcbe6f5866c3cbded3 100644 (file)
--- a/devscripts/make_lazy_extractors.py
+++ b/devscripts/make_lazy_extractors.py
@@ -118,3 +118,14 @@ module_src = '\n'.join(module_contents) + '\n'
 
 with io.open(lazy_extractors_filename, 'wt', encoding='utf-8') as f:
     f.write(module_src)
+
+# work around JVM byte code module limit in Jython
+if sys.platform.startswith('java') and sys.version_info[:2] == (2, 7):
+    import subprocess
+    from youtube_dl.compat import compat_subprocess_get_DEVNULL
+    # if Python 2.7 is available, use it to compile the module for Jython
+    try:
+        # if Python 2.7 is available, use it to compile the module for Jython
+        subprocess.check_call(['python2.7', '-m', 'py_compile', lazy_extractors_filename], stdout=compat_subprocess_get_DEVNULL())
+    except Exception:
+        pass

diff --git a/test/test_http.py b/test/test_http.py
index 1a6b2e87892720809c7978a9dcdb4061c36a5b40..4ec8e13e375586c8e83487315f51ad278c84bd67 100644 (file)
--- a/test/test_http.py
+++ b/test/test_http.py
@@ -45,6 +45,7 @@ from youtube_dl.utils import (
 )
 
 from test.helper import (
+    expectedFailureIf,
     FakeYDL,
     FakeLogger,
     http_server_port,
@@ -243,6 +244,11 @@ class HTTPTestRequestHandler(compat_http_server.BaseHTTPRequestHandler):
 
 
 class TestHTTP(unittest.TestCase):
+    # when does it make sense to check the SSL certificate?
+    _check_cert = (
+        sys.version_info >= (3, 2)
+        or (sys.version_info[0] == 2 and sys.version_info[1:] >= (7, 19)))
+
     def setUp(self):
         # HTTP server
         self.http_httpd = compat_http_server.HTTPServer(
@@ -307,10 +313,7 @@ class TestHTTP(unittest.TestCase):
             else self.https_port if scheme == 'https'
             else self.http_port, path)
 
-    @unittest.skipUnless(
-        sys.version_info >= (3, 2)
-        or (sys.version_info[0] == 2 and sys.version_info[1:] >= (7, 9)),
-        'No support for certificate check in SSL')
+    @unittest.skipUnless(_check_cert, 'No support for certificate check in SSL')
     def test_nocheckcertificate(self):
         with FakeYDL({'logger': FakeLogger()}) as ydl:
             with self.assertRaises(compat_urllib_error.URLError):
@@ -376,6 +379,8 @@ class TestHTTP(unittest.TestCase):
                 with self.assertRaises(compat_urllib_HTTPError):
                     do_req(code, 'GET')
 
+    # Jython 2.7.1 times out for some reason
+    @expectedFailureIf(sys.platform.startswith('java') and sys.version_info < (2, 7, 2))
     def test_content_type(self):
         # https://github.com/yt-dlp/yt-dlp/commit/379a4f161d4ad3e40932dcf5aca6e6fb9715ab28
         with FakeYDL({'nocheckcertificate': True}) as ydl:

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 1d784d90fa652bdc9d0041a4f067fab952694b30..da6d70ec4a1fc89fc01218e52bb03d9ce63524d5 100644 (file)
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -131,7 +131,7 @@ if sys.version_info[0] == 2 or sys.version_info < (3, 3):
         def load(self, rawdata):
             must_have_value = 0
             if not isinstance(rawdata, dict):
-                if sys.version_info[:2] != (2, 7):
+                if sys.version_info[:2] != (2, 7) or sys.platform.startswith('java'):
                     # attribute must have value for parsing
                     rawdata, must_have_value = re.subn(
                         r'(?i)(;\s*)(secure|httponly)(\s*(?:;|$))', r'\1\2=\2\3', rawdata)