Skip to content
This repository has been archived by the owner on Oct 28, 2023. It is now read-only.

Commit

Permalink
Update extracting
Browse files Browse the repository at this point in the history
  • Loading branch information
@HaarigerHarald
HaarigerHarald committed Feb 14, 2018
1 parent 8fb2c7e commit 49b90c0
Show file tree
Hide file tree
Showing 2 changed files with 30 additions and 17 deletions.
10 changes: 5 additions & 5 deletions youtubeExtractor/src/androidTest/java/at/huber/youtubeExtractor/ExtractorCase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ public class ExtractorCase extends InstrumentationTestCase {
private String testUrl;

public void testUsualVideo() throws Throwable {
VideoMeta expMeta = new VideoMeta("YE7VzlLtp-4", "Big Buck Bunny", "Blender Foundation",
VideoMeta expMeta = new VideoMeta("YE7VzlLtp-4", "Big Buck Bunny", "Blender",
"UCSMOQeBJ2RAnuFungnQOxLg", 597, 0, false);
extractorTest("http:https://youtube.com/watch?v=YE7VzlLtp-4", expMeta);
extractorTestDashManifest("http:https://youtube.com/watch?v=YE7VzlLtp-4");
Expand All @@ -26,7 +26,7 @@ public void testUsualVideo() throws Throwable {

public void testEncipheredVideo() throws Throwable {
VideoMeta expMeta = new VideoMeta("e8X3ACToii0", "Rise Against - Savior", "RiseAgainstVEVO",
"UChMKB2AHNpeuWhalpRYhUaw", 244, 0, false);
"UChMKB2AHNpeuWhalpRYhUaw", 243, 0, false);
extractorTest("https://www.youtube.com/watch?v=e8X3ACToii0", expMeta);
}

Expand All @@ -38,9 +38,9 @@ public void testAgeRestrictVideo() throws Throwable {
}

public void testLiveStream() throws Throwable {
VideoMeta expMeta = new VideoMeta("qzMQza8xZCc", "Nasa LIVE stream - Earth From Space LIVE Feed | Incredible ISS live stream of Earth from space",
"Space Videos", "UCakgsb0w7QB0VHdnCc-OVEA", 1800, 0, true);
extractorTest("http:https://www.youtube.com/watch?v=qzMQza8xZCc", expMeta);
VideoMeta expMeta = new VideoMeta("ddFvjfvPnqk", "NASA Live Stream - Earth From Space (Full Screen) | ISS LIVE FEED - Debunk Flat Earth",
"Space Videos", "UCakgsb0w7QB0VHdnCc-OVEA", 0, 0, true);
extractorTest("http:https://www.youtube.com/watch?v=ddFvjfvPnqk", expMeta);
}


Expand Down
37 changes: 25 additions & 12 deletions youtubeExtractor/src/main/java/at/huber/youtubeExtractor/YouTubeExtractor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,13 +57,14 @@ public abstract class YouTubeExtractor extends AsyncTask<String, Void, SparseArr
private final Condition jsExecuting = lock.newCondition();

private static final String USER_AGENT = "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.115 Safari/537.36";
private static final String STREAM_MAP_STRING = "url_encoded_fmt_stream_map";

private static final Pattern patYouTubePageLink = Pattern.compile("(http|https):https://(www\\.|m.|)youtube\\.com/watch\\?v=(.+?)( |\\z|&)");
private static final Pattern patYouTubeShortLink = Pattern.compile("(http|https):https://(www\\.|)youtu.be/(.+?)( |\\z|&)");

private static final Pattern patDashManifest1 = Pattern.compile("dashmpd=(.+?)(&|\\z)");
private static final Pattern patDashManifest2 = Pattern.compile("\"dashmpd\":\"(.+?)\"");
private static final Pattern patDashManifestEncSig = Pattern.compile("/s/([0-9A-F|\\.]{10,}?)(/|\\z)");
private static final Pattern patDashManifestEncSig = Pattern.compile("/s/([0-9A-F|.]{10,}?)(/|\\z)");

private static final Pattern patTitle = Pattern.compile("title=(.*?)(&|\\z)");
private static final Pattern patAuthor = Pattern.compile("author=(.+?)(&|\\z)");
Expand All @@ -74,14 +75,15 @@ public abstract class YouTubeExtractor extends AsyncTask<String, Void, SparseArr
private static final Pattern patHlsvp = Pattern.compile("hlsvp=(.+?)(&|\\z)");
private static final Pattern patHlsItag = Pattern.compile("/itag/(\\d+?)/");

private static final Pattern patItag = Pattern.compile("itag=([0-9]+?)(&|,)");
private static final Pattern patEncSig = Pattern.compile("s=([0-9A-F|\\.]{10,}?)(&|,|\")");
private static final Pattern patUrl = Pattern.compile("url=(.+?)(&|,)");
private static final Pattern patItag = Pattern.compile("itag=([0-9]+?)([&,])");
private static final Pattern patEncSig = Pattern.compile("s=([0-9A-F|.]{10,}?)([&,\"])");
private static final Pattern patIsSigEnc = Pattern.compile("s%3D([0-9A-F|.]{10,}?)%26");
private static final Pattern patUrl = Pattern.compile("url=(.+?)([&,])");

private static final Pattern patVariableFunction = Pattern.compile("(\\{|;| |=)([a-zA-Z$][a-zA-Z0-9$]{0,2})\\.([a-zA-Z$][a-zA-Z0-9$]{0,2})\\(");
private static final Pattern patFunction = Pattern.compile("(\\{|;| |=)([a-zA-Z$_][a-zA-Z0-9$]{0,2})\\(");
private static final Pattern patVariableFunction = Pattern.compile("([{; =])([a-zA-Z$][a-zA-Z0-9$]{0,2})\\.([a-zA-Z$][a-zA-Z0-9$]{0,2})\\(");
private static final Pattern patFunction = Pattern.compile("([{; =])([a-zA-Z$_][a-zA-Z0-9$]{0,2})\\(");
private static final Pattern patDecryptionJsFile = Pattern.compile("jsbin\\\\/(player-(.+?).js)");
private static final Pattern patSignatureDecFunction = Pattern.compile("\\(\"signature\",(.{1,3}?)\\(.{1,10}?\\)");
private static final Pattern patSignatureDecFunction = Pattern.compile("\"signature\",(.{1,3}?)\\(.{1,10}?\\)");

private static final SparseArray<Format> FORMAT_MAP = new SparseArray<>();

Expand Down Expand Up @@ -206,9 +208,11 @@ private SparseArray<YtFile> getStreamUrls() throws IOException, InterruptedExcep
+ URLEncoder.encode("https://youtube.googleapis.com/v/" + videoID, "UTF-8");

String dashMpdUrl = null;
String streamMap = null;
String streamMap;
BufferedReader reader = null;
URL getUrl = new URL(ytInfoUrl);
if(LOGGING)
Log.d(LOG_TAG, "infoUrl: " + ytInfoUrl);
HttpURLConnection urlConnection = (HttpURLConnection) getUrl.openConnection();
urlConnection.setRequestProperty("User-Agent", USER_AGENT);
try {
Expand Down Expand Up @@ -265,10 +269,19 @@ private SparseArray<YtFile> getStreamUrls() throws IOException, InterruptedExcep
return null;
}

// "use_cipher_signature" disappeared, we check whether at least one ciphered signature
// exists int the stream_map.
boolean sigEnc = true;
if(streamMap != null && streamMap.contains(STREAM_MAP_STRING)){
streamMap = streamMap.substring(streamMap.indexOf(STREAM_MAP_STRING));
mat = patIsSigEnc.matcher(streamMap);
if(!mat.find())
sigEnc = false;
}

// Some videos are using a ciphered signature we need to get the
// deciphering js-file from the youtubepage.
if (streamMap == null || !streamMap.contains("use_cipher_signature=False")) {
if (sigEnc) {
// Get the video directly from the youtubepage
if (CACHING
&& (decipherJsFileName == null || decipherFunctions == null || decipherFunctionName == null)) {
Expand All @@ -282,7 +295,7 @@ private SparseArray<YtFile> getStreamUrls() throws IOException, InterruptedExcep
String line;
while ((line = reader.readLine()) != null) {
// Log.d("line", line);
if (line.contains("url_encoded_fmt_stream_map")) {
if (line.contains(STREAM_MAP_STRING)) {
streamMap = line.replace("\\u0026", "&");
break;
}
Expand Down Expand Up @@ -326,7 +339,7 @@ private SparseArray<YtFile> getStreamUrls() throws IOException, InterruptedExcep
streamMap = URLDecoder.decode(streamMap, "UTF-8");
}

streams = streamMap.split(",|url_encoded_fmt_stream_map|&adaptive_fmts=");
streams = streamMap.split(",|"+STREAM_MAP_STRING+"|&adaptive_fmts=");
SparseArray<YtFile> ytFiles = new SparseArray<>();
for (String encStream : streams) {
encStream = encStream + ",";
Expand Down Expand Up @@ -433,7 +446,7 @@ private boolean decipherSignature(final SparseArray<String> encSignatures) throw
String decipherFunctUrl = "https://s.ytimg.com/yts/jsbin/" + decipherJsFileName;

BufferedReader reader = null;
String javascriptFile = null;
String javascriptFile;
URL url = new URL(decipherFunctUrl);
HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection();
urlConnection.setRequestProperty("User-Agent", USER_AGENT);
Expand Down

0 comments on commit 49b90c0

Please sign in to comment.