feat(youtube): add SABR download session helpers

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java

@@ -1509,19 +1509,18 @@ public void onFetchPage(@Nonnull final Downloader downloader)
         CancellableCall webPageCall = YoutubeParsingHelper.getWebPlayerResponse(
                 localization, contentCountry, videoId, this);
 
-        final CancellableCall webSafariPlayerCall = fetchWebSafariJsonPlayer(
-                contentCountry, localization, videoId);
-        final CancellableCall configuredPlayerCall;
+        final CancellableCall jsonPlayerCall;
         switch (NewPipe.getYoutubePlayerClient()) {
             case "web_safari":
-                configuredPlayerCall = null;
+                jsonPlayerCall = fetchWebSafariJsonPlayer(
+                        contentCountry, localization, videoId);
                 break;
             case "web":
-                configuredPlayerCall = fetchWebJsonPlayer(
+                jsonPlayerCall = fetchWebJsonPlayer(
                         contentCountry, localization, videoId);
                 break;
             default:
-                configuredPlayerCall = fetchMwebJsonPlayer(
+                jsonPlayerCall = fetchMwebJsonPlayer(
                         contentCountry, localization, videoId);
                 break;
         }
@@ -1562,8 +1561,7 @@ public void onSuccess(Response response) throws ExtractionException {
             }
             long startTime = System.nanoTime();
             do {
-                if (webSafariPlayerCall.isFinished()
-                        && (configuredPlayerCall == null || configuredPlayerCall.isFinished())
+                if (jsonPlayerCall.isFinished()
                         && webPageCall.isFinished() && nextDataCall.isFinished()) {
                     break;
                 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrDecodedResponse.java

@@ -396,6 +396,10 @@ public boolean isProtectedNoMediaResponse() {
         return isNoMediaResponse() && streamProtectionStatus >= 3;
     }
 
+    public boolean isProtectionBoundaryNoMediaResponse() {
+        return isNoMediaResponse() && streamProtectionStatus >= 2;
+    }
+
     @Nonnull
     public String summarizeNoMediaResponse() {
         return "parts=" + parts.size()

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrMp4SegmentIndexParser.java

@@ -15,26 +15,41 @@ private SabrMp4SegmentIndexParser() {
     static SabrSegmentIndex parse(@Nonnull final byte[] initData,
                                   @Nonnull final SabrFormatInitializationMetadata metadata)
             throws SabrProtocolException {
-        final int indexStart = checkedRangeOffset(metadata.getIndexRangeStart(), initData.length);
-        final int indexEnd = checkedRangeOffset(metadata.getIndexRangeEnd(), initData.length);
+        return parse(initData,
+                checkedRangeOffset(metadata.getIndexRangeStart(), initData.length),
+                checkedRangeOffset(metadata.getIndexRangeEnd(), initData.length));
+    }
+
+    @Nonnull
+    static SabrSegmentIndex parse(@Nonnull final byte[] initData,
+                                  @Nonnull final YoutubeSabrFormat format)
+            throws SabrProtocolException {
+        return parse(initData, 0, initData.length - 1);
+    }
+
+    @Nonnull
+    private static SabrSegmentIndex parse(@Nonnull final byte[] initData,
+                                          final int indexStart,
+                                          final int indexEnd)
+            throws SabrProtocolException {
         if (indexEnd < indexStart) {
             throw new SabrProtocolException("Invalid MP4 SIDX range");
         }
         final int sidxOffset = findSidxBox(initData, indexStart, indexEnd + 1);
-        return parse(initData, sidxOffset, indexEnd + 1);
+        return parseSidx(initData, sidxOffset, indexEnd + 1);
     }
 
     @Nonnull
     static SabrSegmentIndex parse(@Nonnull final byte[] initData)
             throws SabrProtocolException {
         final int sidxOffset = findSidxBox(initData, 0, initData.length);
-        return parse(initData, sidxOffset, initData.length);
+        return parseSidx(initData, sidxOffset, initData.length);
     }
 
     @Nonnull
-    private static SabrSegmentIndex parse(@Nonnull final byte[] initData,
-                                          final int sidxOffset,
-                                          final int rangeEnd)
+    private static SabrSegmentIndex parseSidx(@Nonnull final byte[] initData,
+                                              final int sidxOffset,
+                                              final int rangeEnd)
             throws SabrProtocolException {
         final long boxSize = readUint32(initData, sidxOffset);
         final int boxEnd = checkedBoxEnd(sidxOffset, boxSize, rangeEnd);

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrWebmSegmentIndexParser.java

@@ -20,38 +20,36 @@ private SabrWebmSegmentIndexParser() {
     static SabrSegmentIndex parse(@Nonnull final byte[] initData,
                                   @Nonnull final SabrFormatInitializationMetadata metadata)
             throws SabrProtocolException {
-        final Element segment = findElement(initData, 0, initData.length, SEGMENT_ID);
-        final long timecodeScaleNanos = readTimecodeScale(initData, segment);
-        final Element cues = findElement(initData,
-                checkedRangeOffset(metadata.getIndexRangeStart(), initData.length),
-                checkedRangeEnd(metadata.getIndexRangeEnd(), initData.length), CUES_ID);
-        final long totalDurationMs = metadata.getDurationUnits() > 0
+        return parse(initData, metadata, metadata.getDurationUnits() > 0
                 && metadata.getDurationTimescale() > 0
                 ? scaleToMs(metadata.getDurationUnits(), metadata.getDurationTimescale())
-                : -1;
-        return parse(initData, cues, timecodeScaleNanos, totalDurationMs);
+                : -1);
     }
 
     @Nonnull
     static SabrSegmentIndex parse(@Nonnull final byte[] initData,
-                                  final long totalDurationMs)
+                                  @Nonnull final YoutubeSabrFormat format)
             throws SabrProtocolException {
-        final Element segment = findElement(initData, 0, initData.length, SEGMENT_ID);
-        final long timecodeScaleNanos = readTimecodeScale(initData, segment);
-        final Element cues = findElement(initData, 0, initData.length, CUES_ID);
-        return parse(initData, cues, timecodeScaleNanos, totalDurationMs);
+        return parse(initData, null, format.getApproxDurationMs());
     }
 
     @Nonnull
     private static SabrSegmentIndex parse(@Nonnull final byte[] initData,
-                                          @Nonnull final Element cues,
-                                          final long timecodeScaleNanos,
+                                          final SabrFormatInitializationMetadata metadata,
                                           final long totalDurationMs)
             throws SabrProtocolException {
+        final Element segment = findElement(initData, 0, initData.length, SEGMENT_ID);
+        final long timecodeScaleNanos = readTimecodeScale(initData, segment);
+        final Element cues = metadata == null
+                ? findElement(initData, segment.contentStart, segment.contentEnd, CUES_ID)
+                : findElement(initData,
+                        checkedRangeOffset(metadata.getIndexRangeStart(), initData.length),
+                        checkedRangeEnd(metadata.getIndexRangeEnd(), initData.length), CUES_ID);
         final List<Long> cueTimes = readCueTimes(initData, cues, timecodeScaleNanos);
         if (cueTimes.isEmpty()) {
             throw new SabrProtocolException("WebM cues contain no cue times");
         }
+
         final List<SabrSegmentIndex.Entry> entries = new ArrayList<>(cueTimes.size());
         for (int i = 0; i < cueTimes.size(); i++) {
             final long startMs = cueTimes.get(i);

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/YoutubeSabrFormat.java

@@ -152,13 +152,8 @@ private static String decodeStreamingUrl(@Nonnull final String videoId,
                 final String signature = YoutubeJavaScriptPlayerManager
                         .deobfuscateSignature(videoId, obfuscatedSignature);
                 final String separator = url.contains("?") ? "&" : "?";
-                try {
-                    url = url + separator + URLEncoder.encode(signatureParameter,
-                            StandardCharsets.UTF_8.name())
-                            + '=' + URLEncoder.encode(signature, StandardCharsets.UTF_8.name());
-                } catch (final UnsupportedEncodingException e) {
-                    throw new ParsingException("Could not encode signature", e);
-                }
+                url = url + separator + urlEncode(signatureParameter) + '='
+                        + urlEncode(signature);
             }
         }
         return YoutubeSabrProbe.maybeDeobfuscateNParameter(videoId, url);
@@ -177,19 +172,30 @@ private static Map<String, String> parseQuery(@Nullable final String value)
             if (equals <= 0) {
                 continue;
             }
-            try {
-                final String key = URLDecoder.decode(part.substring(0, equals),
-                        StandardCharsets.UTF_8.name());
-                final String decodedValue = URLDecoder.decode(part.substring(equals + 1),
-                        StandardCharsets.UTF_8.name());
-                params.put(key, decodedValue);
-            } catch (final UnsupportedEncodingException e) {
-                throw new ParsingException("Could not decode query", e);
-            }
+            params.put(urlDecode(part.substring(0, equals)),
+                    urlDecode(part.substring(equals + 1)));
         }
         return params;
     }
 
+    @Nonnull
+    private static String urlEncode(@Nonnull final String value) throws ParsingException {
+        try {
+            return URLEncoder.encode(value, StandardCharsets.UTF_8.name());
+        } catch (final UnsupportedEncodingException e) {
+            throw new ParsingException("Could not encode SABR signature cipher", e);
+        }
+    }
+
+    @Nonnull
+    private static String urlDecode(@Nonnull final String value) throws ParsingException {
+        try {
+            return URLDecoder.decode(value, StandardCharsets.UTF_8.name());
+        } catch (final UnsupportedEncodingException e) {
+            throw new ParsingException("Could not decode SABR signature cipher", e);
+        }
+    }
+
     private static long parseLong(@Nullable final Object value) {
         if (value instanceof Number) {
             return ((Number) value).longValue();

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/YoutubeSabrProbe.java

@@ -533,20 +533,14 @@ static String maybeDeobfuscateNParameter(@Nonnull final String videoId,
         final java.util.regex.Matcher queryMatcher = java.util.regex.Pattern.compile("([?&])n=([^&]+)")
                 .matcher(url);
         if (queryMatcher.find()) {
-            try {
-                final String encryptedN = java.net.URLDecoder.decode(queryMatcher.group(2),
-                        StandardCharsets.UTF_8.name());
-                final org.schabi.newpipe.extractor.services.youtube.YoutubeApiDecoder.BatchDecodeResult result =
-                        YoutubeJavaScriptPlayerManager.deobfuscateBatch(videoId, null,
-                                Collections.singletonList(encryptedN));
-                final String decryptedN = result.getNParameters().get(encryptedN);
-                if (decryptedN != null) {
-                    return url.substring(0, queryMatcher.start(2))
-                            + java.net.URLEncoder.encode(decryptedN, StandardCharsets.UTF_8.name())
-                            + url.substring(queryMatcher.end(2));
-                }
-            } catch (final UnsupportedEncodingException e) {
-                throw new ParsingException("Could not decode n parameter", e);
+            final String encryptedN = urlDecode(queryMatcher.group(2));
+            final org.schabi.newpipe.extractor.services.youtube.YoutubeApiDecoder.BatchDecodeResult result =
+                    YoutubeJavaScriptPlayerManager.deobfuscateBatch(videoId, null,
+                            Collections.singletonList(encryptedN));
+            final String decryptedN = result.getNParameters().get(encryptedN);
+            if (decryptedN != null) {
+                return url.substring(0, queryMatcher.start(2)) + urlEncode(decryptedN)
+                        + url.substring(queryMatcher.end(2));
             }
         }
 
@@ -562,4 +556,22 @@ static String maybeDeobfuscateNParameter(@Nonnull final String videoId,
         final String decryptedN = result.getNParameters().get(encryptedN);
         return decryptedN == null ? url : url.replace("/n/" + encryptedN, "/n/" + decryptedN);
     }
+
+    @Nonnull
+    private static String urlEncode(@Nonnull final String value) throws ParsingException {
+        try {
+            return java.net.URLEncoder.encode(value, StandardCharsets.UTF_8.name());
+        } catch (final UnsupportedEncodingException e) {
+            throw new ParsingException("Could not encode SABR URL parameter", e);
+        }
+    }
+
+    @Nonnull
+    private static String urlDecode(@Nonnull final String value) throws ParsingException {
+        try {
+            return java.net.URLDecoder.decode(value, StandardCharsets.UTF_8.name());
+        } catch (final UnsupportedEncodingException e) {
+            throw new ParsingException("Could not decode SABR URL parameter", e);
+        }
+    }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/YoutubeSabrSession.java

@@ -138,7 +138,7 @@ public SabrMediaSegment fetchSegment(@Nonnull final SabrSegmentRequest request,
                         + describeRequest(request) + " (reload budget spent): "
                         + decoded.summarizeNoMediaResponse());
             }
-            if (decoded.isProtectedNoMediaResponse()) {
+            if (decoded.isProtectionBoundaryNoMediaResponse()) {
                 if (applyPoTokenForProtectedResponse()) {
                     if (decoded.getBackoffTimeMs() > 0) {
                         sleepBackoff(decoded.getBackoffTimeMs());
@@ -265,10 +265,10 @@ public List<SabrMediaSegment> pumpOnce(@Nonnull final Localization localization)
             throw new SabrProtocolException("SABR requested player reload (reload budget spent): "
                     + decoded.summarizeNoMediaResponse());
         }
-        if (decoded.isProtectedNoMediaResponse()) {
-            // mint / re-mint the token, best effort. don't throw on a single status=3: it's normal
-            // pacing, the server usually clears it next round. pump keeps trying; the stall watchdog
-            // is the real give-up.
+        if (decoded.isProtectionBoundaryNoMediaResponse()) {
+            // Mint / re-mint the token as soon as SABR reaches the protection boundary. Do not throw
+            // on a single no-media round: the server usually clears it next round. The pump keeps
+            // trying; the stall watchdog is the real give-up.
             applyPoTokenForProtectedResponse();
         }
         if (!segments.isEmpty()) {
@@ -372,6 +372,15 @@ public SabrMediaSegment getCachedSegment(@Nonnull final SabrSegmentRequest reque
         return segmentCache.get(cacheKey(request));
     }
 
+    public void discardCachedSegment(@Nonnull final SabrSegmentRequest request) {
+        final String key = cacheKey(request);
+        final SabrMediaSegment removed = segmentCache.remove(key);
+        if (removed != null && !removed.getHeader().isInitSegment()) {
+            cacheOrder.remove(key);
+            cachedBytes = Math.max(0, cachedBytes - removed.getLength());
+        }
+    }
+
 
     /** True once the requested media segment is known to be past the last segment of the stream. */
     public boolean isBeyondEnd(@Nonnull final SabrSegmentRequest request) {