From 1dc3be801b475aeee32fb984f13281595a20086f Mon Sep 17 00:00:00 2001 From: w33b Date: Sun, 8 Feb 2026 12:10:28 +0100 Subject: [PATCH] Refactor & Rewrite --- Start.py | 59 ++++++----- utils/encodeCDN.py | 183 --------------------------------- utils/encodeDDL.py | 52 ---------- utils/encode_downloads.py | 85 ++++++++++++++++ utils/encode_stream.py | 209 ++++++++++++++++++++++++++++++++++++++ utils/interpolate.py | 107 ++++++++++++------- utils/interpolate4k.py | 42 -------- utils/mediainfo.py | 39 +++---- utils/upcale.py | 102 ++++++++++++++----- 9 files changed, 493 insertions(+), 385 deletions(-) delete mode 100644 utils/encodeCDN.py delete mode 100644 utils/encodeDDL.py create mode 100644 utils/encode_downloads.py create mode 100644 utils/encode_stream.py delete mode 100644 utils/interpolate4k.py diff --git a/Start.py b/Start.py index 55f9ba7..1491ddc 100644 --- a/Start.py +++ b/Start.py @@ -1,43 +1,50 @@ import os import re -from utils.encodeCDN import EncodeCDN -from utils.encodeDDL import EncodeDDL -from utils.interpolate import Interpolate -from utils.interpolate4k import Interpolate4K +from utils.encode_downloads import encode_downloads +from utils.encode_stream import encode_streams +from utils.interpolate import interpolate from utils.upcale import upscale from utils.mediainfo import get_aspect_ratio INTERPOLATE_4K = False -MAX_INPUT_WIDTH = '720' def create_folder(folder_path): if not os.path.exists(folder_path): os.makedirs(folder_path) for filename in os.listdir('0-Source'): - input_file = os.path.join('0-Source', filename) - if not os.path.isfile(input_file): - continue if filename == '.gitignore': continue + input_file = os.path.join('0-Source', filename) + if not os.path.isfile(input_file): + continue + # Parse File Name - temp_name = re.sub(r'\[.*?\]|\(.*?\)', "", filename).rsplit('.', 1)[0].strip() - folder_name = re.sub(r'[^A-Za-z ]+', '', temp_name).strip() - episode_number = re.findall(r'\d+', temp_name)[-1] + try: + hentai_name = re.sub(r'\[.*?\]|\(.*?\)', "", filename).rsplit('.', 1)[0].strip() + folder_name = re.sub(r'[^A-Za-z ]+', '', hentai_name).strip() + episode_number = re.findall(r'\d+', hentai_name)[-1] + except: + print(f"Error Parsing name for: {filename}") + continue cdn_folder_name = folder_name.replace(" ", ".") - cdn_folder = os.path.join('2-Out', folder_name, cdn_folder_name, 'E' + episode_number) + cdn_folder = os.path.join('2-Out', folder_name, cdn_folder_name, f'E{episode_number}') muxed_folder = os.path.join('2-Out', folder_name, 'Muxed') - upscale_output_folder = os.path.join('2-Out', folder_name, folder_name + ' [2160p]') - upscale_output = os.path.join(upscale_output_folder, temp_name + ' [4k][HEVC].mkv') - - interpolate_output = os.path.join(upscale_output_folder, temp_name + ' [1080p][48fps][HEVC].mkv') - interpolate_4k_output = os.path.join(upscale_output_folder, temp_name + ' [2160p][48fps][HEVC].mkv') + # Output of Upscale and Interpolation + # Hentai Name [2160p] + # -> "Hentai Name [4k][HEVC].mkv" + # -> "Hentai Name [1080p][48fps][HEVC].mkv" + # -> "Hentai Name [2160p][48fps][HEVC].mkv" + upscale_output_folder = os.path.join('2-Out', folder_name, f'{folder_name} [2160p]') + upscale_output = os.path.join(upscale_output_folder, f'{hentai_name} [4k][HEVC].mkv') + interpolate_output = os.path.join(upscale_output_folder, f'{hentai_name} [1080p][48fps][HEVC].mkv') + interpolate_4k_output = os.path.join(upscale_output_folder, f'{hentai_name} [2160p][48fps][HEVC].mkv') - print('Parsed Name: ' + temp_name) + print('Parsed Name: ' + hentai_name) create_folder(cdn_folder) create_folder(muxed_folder) @@ -45,8 +52,14 @@ for filename in os.listdir('0-Source'): aspect_ratio = get_aspect_ratio(input_file) - upscale(input_file, upscale_output, MAX_INPUT_WIDTH, aspect_ratio) - Interpolate(interpolate_output, upscale_output, temp_name, aspect_ratio) - Interpolate4K(interpolate_4k_output, upscale_output, INTERPOLATE_4K, temp_name) - EncodeDDL(input_file, cdn_folder, folder_name, temp_name, upscale_output, aspect_ratio, interpolate_output, INTERPOLATE_4K, interpolate_4k_output) - EncodeCDN(input_file, cdn_folder, aspect_ratio, upscale_output, interpolate_output, INTERPOLATE_4K, interpolate_4k_output) + upscale(input_file, upscale_output, aspect_ratio) + interpolate(upscale_output, interpolate_output, hentai_name, "1080p", aspect_ratio) + + if INTERPOLATE_4K: + interpolate(upscale_output, interpolate_4k_output, hentai_name, "2160p", aspect_ratio) + encode_downloads(folder_name, hentai_name, input_file, upscale_output, interpolate_output, interpolate_4k_output, aspect_ratio) + encode_streams(cdn_folder, input_file, upscale_output, interpolate_output, interpolate_4k_output, aspect_ratio) + else: + encode_downloads(folder_name, hentai_name, input_file, upscale_output, interpolate_output, None, aspect_ratio) + encode_streams(cdn_folder, input_file, upscale_output, interpolate_output, None, aspect_ratio) + diff --git a/utils/encodeCDN.py b/utils/encodeCDN.py deleted file mode 100644 index 3d3713c..0000000 --- a/utils/encodeCDN.py +++ /dev/null @@ -1,183 +0,0 @@ -import os -import subprocess -import shutil -import platform - -segment = ' -init_seg_name chunks/init-stream$RepresentationID$.m4s -media_seg_name chunks/chunk-stream$RepresentationID$-$Number%05d$.m4s' -if platform.system() == 'Linux': - segment = ' -init_seg_name chunks/init-stream\$RepresentationID\$.m4s -media_seg_name chunks/chunk-stream\$RepresentationID\$-\$Number%05d\$.m4s' - -def changeM4SToHTML(mpdpath, chunkspath): - for filename in os.listdir(chunkspath): - infilename = os.path.join(chunkspath, filename) - if not os.path.isfile(infilename): continue - newname = infilename.replace('.m4s', '.webp') - _ = os.rename(infilename, newname) - - # Modify manifest - with open(mpdpath, 'r') as file : - filedata = file.read() - # Replace the target string - filedata = filedata.replace('.m4s', '.webp') - # Write the file out again - with open(mpdpath, 'w') as file: - file.write(filedata) - -def createFolder(cdnFolder, resolution): - if platform.system() == 'Linux': - if not os.path.exists(os.path.join(cdnFolder, resolution, 'chunks')): - os.makedirs(os.path.join(cdnFolder, resolution, 'chunks')) - else: - if not os.path.exists('chunks'): - os.makedirs('chunks') - - if not os.path.exists(os.path.join(cdnFolder, resolution)): - os.makedirs(os.path.join(cdnFolder, resolution)) - -def create_sprites(cdn_folder): - """ - Creates video player sprites - """ - video_file = os.path.join(cdn_folder, 'x264.720p.mp4') - # Generating Sprites - if not os.path.exists(os.path.join(cdn_folder, 'thumbs.vtt')) and os.path.exists(video_file): - os.system(f'python makesprites.py "{video_file}"') - os.rename("thumbs.vtt", os.path.join(cdn_folder, 'thumbs.vtt')) - os.rename("sprite.jpg", os.path.join(cdn_folder, 'sprite.jpg')) - shutil.rmtree('thumbs') - shutil.rmtree('logs') - return - - print('Skipped Sprites') - -def encode_720p_fallback(cdn_folder, video_source, upscale_output, aspect_ratio): - """ - Fallback video stream for apple devices - """ - output = os.path.join(cdn_folder, 'x264.720p.mp4') - - if os.path.exists(output): - print('Skipped 720p Encode') - return - - command = ( - f'ffmpeg -v quiet -stats -i "{upscale_output}" -i "{video_source}" ' - '-map 0:v:0 -map 1:a:0 ' - '-c:v libx264 -crf 22 -pix_fmt yuv420p ' - f'-vf scale=1280:720,setsar=1:1 -aspect {aspect_ratio} ' - '-c:a aac -b:a 128k ' - f'-sn -map_metadata -1 -movflags +faststart "{output}"' - ) - - subprocess.call(command, shell=True) - -def encode_720p(cdn_folder, video_source, upscale_output, aspect_ratio): - output = os.path.join(cdn_folder, '720', 'manifest.mpd') - - if os.path.exists(output): - print('Skipped 720p DASH Encode') - return - - print('Encoding 720p') - - createFolder(cdn_folder, '720') - - command = ( - f'ffmpeg -v quiet -stats -i "{upscale_output}" -i "{video_source}" ' - '-map 0:v:0 -map 1:a:0 ' - '-c:v libx264 -crf 22 -preset medium -pix_fmt yuv420p -g 24 -keyint_min 24 -sc_threshold 0 -x264-params keyint=24:min-keyint=24:scenecut=0 ' - f'-vf scale=1280:720,setsar=1:1 -aspect {aspect_ratio} ' - f'{segment} "{output}"' - ) - - subprocess.call(command, shell=True) - - if platform.system() != 'Linux': - shutil.move('chunks', os.path.join(cdn_folder, '720', 'chunks')) - - changeM4SToHTML(output, os.path.join(cdn_folder, '720', 'chunks')) - -def EncodeCDN(f, cdnFolder, inputAspect, upscaleOut, interpolateOut, INTERPOLATE_4K, interpolate_4k_output): - - out1080mpd = os.path.join(cdnFolder, '1080', 'manifest.mpd') - out1080mpd48 = os.path.join(cdnFolder, '1080i', 'manifest.mpd') - out2160mpd = os.path.join(cdnFolder, '2160', 'manifest.mpd') - out2160mpd48 = os.path.join(cdnFolder, '2160i', 'manifest.mpd') - - # 720p - encode_720p(cdnFolder, f, upscaleOut, inputAspect) - encode_720p_fallback(cdnFolder, f, upscaleOut, inputAspect) - create_sprites(cdnFolder) - - # 1080p - if not os.path.exists(out1080mpd): - print('Encoding 1080p') - - createFolder(cdnFolder, '1080') - - subprocess.call('ffmpeg -v quiet -stats -i "' + upscaleOut + '" -i "' + f + '" -map 0:v:0 -map 1:a:0' - + ' -c:v libsvtav1 -preset 6 -crf 26 -pix_fmt yuv420p -svtav1-params keyint=1s:tune=0 -vf scale=1920:1080,setsar=1:1 -aspect ' + inputAspect - + ' -c:a aac -b:a 128k -ac 2 -sn -map_metadata -1 -seg_duration 10 -use_template 1 -use_timeline 1' - + segment + ' "' + out1080mpd + '"', shell=True) - - if platform.system() != 'Linux': - shutil.move('chunks', os.path.join(cdnFolder, '1080', 'chunks')) - - changeM4SToHTML(out1080mpd, os.path.join(cdnFolder, '1080', 'chunks')) - else: - print('Skipped 1080p Encode') - - # 1080p 48fps - if not os.path.exists(out1080mpd48): - print('Encoding 1080p 48fps') - - createFolder(cdnFolder, '1080i') - - # DASH av1 1080p - subprocess.call('ffmpeg -v quiet -stats -i "' + interpolateOut + '" -i "' + f + '" -map 0:v:0 -map 1:a:0' - + ' -c:v libsvtav1 -preset 6 -crf 26 -pix_fmt yuv420p -svtav1-params keyint=1s:tune=0 -vf scale=1920:1080,setsar=1:1 -aspect ' + inputAspect - + ' -c:a aac -b:a 128k -ac 2 -sn -map_metadata -1 -seg_duration 10 -use_template 1 -use_timeline 1' - + segment + ' "' + out1080mpd48 + '"', shell=True) - - if platform.system() != 'Linux': - shutil.move('chunks', os.path.join(cdnFolder, '1080i', 'chunks')) - - changeM4SToHTML(out1080mpd48, os.path.join(cdnFolder, '1080i', 'chunks')) - else: - print('Skipped 1080p 48fps Encode') - - # 2160p - if not os.path.exists(out2160mpd): - print('Encoding 2160p') - - createFolder(cdnFolder, '2160') - - subprocess.call('ffmpeg -v quiet -stats -i "' + upscaleOut + '" -i "' + f + '" -map 0:v:0 -map 1:a:0' - + ' -c:v libsvtav1 -preset 6 -crf 26 -pix_fmt yuv420p -svtav1-params keyint=1s:tune=0 -vf "scale=\'min(3840,iw)\':-2,setsar=1:1" -aspect ' + inputAspect - + ' -c:a aac -b:a 128k -ac 2 -sn -map_metadata -1 -seg_duration 10 -use_template 1 -use_timeline 1' - + segment + ' "' + out2160mpd + '"', shell=True) - - if platform.system() != 'Linux': - shutil.move('chunks', os.path.join(cdnFolder, '2160', 'chunks')) - - changeM4SToHTML(out2160mpd, os.path.join(cdnFolder, '2160', 'chunks')) - else: - print('Skipped 2160p Encode') - - # 2160p 48fps - if not os.path.exists(out2160mpd48) and INTERPOLATE_4K: - print('Encoding 2160p 48fps') - - createFolder(cdnFolder, '2160i') - - subprocess.call('ffmpeg -v quiet -stats -i "' + interpolate_4k_output + '" -i "' + f + '" -map 0:v:0 -map 1:a:0' - + ' -c:v libsvtav1 -preset 6 -crf 26 -pix_fmt yuv420p -svtav1-params keyint=1s:tune=0 -vf "scale=\'min(3840,iw)\':-2,setsar=1:1" -aspect ' + inputAspect - + ' -c:a aac -b:a 128k -ac 2 -sn -map_metadata -1 -seg_duration 10 -use_template 1 -use_timeline 1' - + segment + ' "' + out2160mpd48 + '"', shell=True) - - if platform.system() != 'Linux': - shutil.move('chunks', os.path.join(cdnFolder, '2160i', 'chunks')) - - changeM4SToHTML(out2160mpd48, os.path.join(cdnFolder, '2160i', 'chunks')) - else: - print('Skipped 2160p 48fps Encode') diff --git a/utils/encodeDDL.py b/utils/encodeDDL.py deleted file mode 100644 index 67a0b03..0000000 --- a/utils/encodeDDL.py +++ /dev/null @@ -1,52 +0,0 @@ -import os -import subprocess - -def extract_subs(video_source, subtitle_out, vtt_out): - if os.path.exists(subtitle_out): - print('Skipped Sub Extract') - return - print('Extracting Sub') - subprocess.call(f'ffmpeg -v quiet -stats -i "{video_source}" -c copy "{subtitle_out}"', shell=True) - subprocess.call(f'ffmpeg -v quiet -stats -i "{subtitle_out}" "{vtt_out}"', shell=True) - -def encode_video(video_source, input_file, output_file, mux_file, temp_name, input_aspect, width, height): - if os.path.exists(output_file): - print(f'Skipped {height}p HEVC Encode') - return - - print(f'Encoding {height}p HEVC') - command = (f'ffmpeg -v quiet -stats -i "{input_file}" -i "{video_source}"' - ' -map 0:v:0 -map 1:a:0 -map 1:s:0 -map 1:t? -map 1:d?' - f' -disposition:v:0 default -metadata Title="{temp_name} [hstream.moe]"' - ' -metadata:s:v:0 title="Upscaled by hstream.moe"' - ' -c:v hevc_nvenc -qp 18 -pix_fmt yuv420p10le' - f' -vf "scale=\'min({width},iw)\':-2,setsar=1:1" -aspect {input_aspect}' - ' -c:a aac -b:a 160k -c:s copy' - f' "{output_file}"' - ) - - subprocess.call(command, shell=True) - subprocess.run(f'mkvmerge --output "{mux_file}" "{output_file}"', shell=True) - - -def EncodeDDL(video_source, cdn_folder, folder_name, temp_name, upscale_out, input_aspect, interpolate_out, INTERPOLATE_4K, interpolate_4k_output): - # Extract subtitles - out_ass = os.path.join(cdn_folder, 'eng.ass') - out_vtt = os.path.join(cdn_folder, 'eng.vtt') - extract_subs(video_source, out_ass, out_vtt) - - # Encoding settings - resolutions = [ - (1920, 1080, upscale_out, "[1080p-HEVC]"), - (1920, 1080, interpolate_out, "[1080p-HEVC][48fps]"), - (3840, 2160, upscale_out, "[2160p-HEVC]") - ] - - # Also encode 4k 48fps if enabled - if INTERPOLATE_4K: - resolutions.append((3840, 2160, interpolate_4k_output, "[2160p-HEVC][48fps]")) - - for width, height, input_file, suffix in resolutions: - tmp_out = os.path.join('2-Out', folder_name, f"{temp_name} {suffix}[hstream.moe].mkv") - mux_out = os.path.join('2-Out', folder_name, 'Muxed', f"{temp_name} {suffix}[hstream.moe].mkv") - encode_video(video_source, input_file, tmp_out, mux_out, temp_name, input_aspect, width, height) diff --git a/utils/encode_downloads.py b/utils/encode_downloads.py new file mode 100644 index 0000000..ad83058 --- /dev/null +++ b/utils/encode_downloads.py @@ -0,0 +1,85 @@ +import os +import sys +import subprocess + +def _remux_video( + encoded_video_input: str, + muxed_video_output: str, +): + # Re-Mux with MKVMerge, as FFmpeg has some kind of bug + try: + subprocess.run(f'mkvmerge --output "{muxed_video_output}" "{encoded_video_input}"', shell=True, check=True) + except subprocess.CalledProcessError as e: + print(f"\nffmpeg failed with error code {e.returncode}", file=sys.stderr) + sys.exit(e.returncode) + + +def _encode_video( + preset: dict[str, str], + source_video: str, + output_video: str, + hentai_title: str, + input_aspect: str = "16:9" +): + print(f'Encoding {preset['h']}p AV1') + + cmd = [ + "ffmpeg", + "-i", preset['input_video'], + "-i", source_video, + "-map", "0:v:0", # Video from upscale or interpolated file + "-map", "1:a:0", # Audio from source video + "-map", "1:s:0", # Subtitle from source video + "-map", "1:t?", # Attachments from source video (optional) + "-map", "1:d?", # Other Data from source video (optional) + "-disposition:v:0", "default", # Mark video as default in mkv container + "-metadata", f'Title=\"{hentai_title} [hstream.moe]\"', + "-c:v", "libsvtav1", + "-crf", preset['crf'], + "-preset", "4", + "-pix_fmt", "yuv420p10le", # 10bit + "-vf", f"scale=\'min({preset['w']},iw)\':-2,setsar=1:1", + "-aspect", input_aspect, + "-c:a", "libopus", + "-b:a", "160k", + "-c:s", "copy", + output_video + ] + + print(cmd) + + try: + subprocess.run(cmd, shell=True, check=True) + except subprocess.CalledProcessError as e: + print(f"\nffmpeg failed with error code {e.returncode}", file=sys.stderr) + sys.exit(e.returncode) + +def encode_downloads( + folder_name: str, + hentai_title: str, + source_video: str, + upscaled_video: str, + interpolated_video: str, + interpolated_uhd_video: str | None, + input_aspect: str = "16:9" +): + presets = [ + {"w": "1920", "h": "1080", "crf": "22", "name": "[1080p-AV1]", "input_video": upscaled_video}, + {"w": "1920", "h": "1080", "crf": "22", "name": "[1080p-AV1][48fps]", "input_video": interpolated_video}, + {"w": "3840", "h": "2160", "crf": "24", "name": "[2160p-AV1]", "input_video": upscaled_video}, + ] + + if interpolated_uhd_video is not None: + presets.append({"w": "3840", "h": "2160", "crf": "24", "name": "[2160p-AV1][48fps]", "input_video": interpolated_uhd_video}) + + for preset in presets: + file_name = f"{hentai_title} {preset['name']}[hstream.moe].mkv" + tmp_out = os.path.join('2-Out', folder_name, file_name) + mux_out = os.path.join('2-Out', folder_name, 'Muxed', file_name) + + if os.path.exists(mux_out): + print(f'Skipped {preset['h']}p AV1 Encode') + return + + _encode_video(preset, source_video, tmp_out, hentai_title, input_aspect) + _remux_video(tmp_out, mux_out) diff --git a/utils/encode_stream.py b/utils/encode_stream.py new file mode 100644 index 0000000..9d71fb9 --- /dev/null +++ b/utils/encode_stream.py @@ -0,0 +1,209 @@ +import os +import subprocess +import shutil +import platform +import sys + +def _extract_subs( + source_video: str, + cdn_folder: str +): + out_ass = os.path.join(cdn_folder, 'eng.ass') + out_vtt = os.path.join(cdn_folder, 'eng.vtt') + + if os.path.exists(out_ass): + print('Skipped Sub Extract') + return + + print('Extracting Sub') + subprocess.call(f'ffmpeg -y -v quiet -stats -i "{source_video}" -c copy "{out_ass}"', shell=True) + subprocess.call(f'ffmpeg -y -v quiet -stats -i "{out_ass}" "{out_vtt}"', shell=True) + +def _create_sprites(cdn_folder: str): + """ + Creates video player sprites + """ + video_file = os.path.join(cdn_folder, 'x264.720p.mp4') + # Generating Sprites + if not os.path.exists(os.path.join(cdn_folder, 'thumbs.vtt')) and os.path.exists(video_file): + os.system(f'python makesprites.py "{video_file}"') + os.rename("thumbs.vtt", os.path.join(cdn_folder, 'thumbs.vtt')) + os.rename("sprite.jpg", os.path.join(cdn_folder, 'sprite.jpg')) + shutil.rmtree('thumbs') + shutil.rmtree('logs') + return + + print('Skipped Sprites') + +def _encode_720p_fallback( + cdn_folder: str, + video_source: str, + upscale_output: str, + aspect_ratio: str = "16:9" +): + """ + Fallback video stream for apple devices + """ + output = os.path.join(cdn_folder, 'x264.720p.mp4') + + if os.path.exists(output): + print('Skipped 720p Encode') + return + + cmd = [ + "ffmpeg", "-v", "quiet", "-stats", + "-i", upscale_output, + "-i", video_source, + "-map", "0:v:0", + "-map", "1:a:0", + "-c:v", "libx264", + "-pix_fmt", "yuv420p", + "-vf", "scale=1280:720,setsar=1:1", + "-aspect", aspect_ratio, + "-c:a", "aac", + "-b:a", "128k", + "-sn", + "-map_metadata", "-1", + "-movflags", "+faststart", + output + ] + + try: + subprocess.run(cmd, shell=True, check=True) + except subprocess.CalledProcessError as e: + print(f"\nffmpeg failed with error code {e.returncode}", file=sys.stderr) + sys.exit(e.returncode) + + +def _change_chunk_extension( + preset: dict[str, str], + cdn_folder: str, +): + chunks_folder = os.path.join(cdn_folder, preset['out_folder'], 'chunks') + mpd_file = os.path.join(cdn_folder, preset['out_folder'], 'manifest.mpd') + + # Move encodes on Windows to correct folder + if platform.system() != 'Linux': + shutil.move('chunks', chunks_folder) + + # Rename files + for filename in os.listdir(chunks_folder): + file_path = os.path.join(chunks_folder, filename) + if not os.path.isfile(file_path): + continue + + new_file_path = file_path.replace('.webm', '.webp') + os.rename(file_path, new_file_path) + + # Modify manifest + with open(mpd_file, 'r') as file : + filedata = file.read() + + # Replace the target string + filedata = filedata.replace('.webm', '.webp') + + # Write the file out again + with open(mpd_file, 'w') as file: + file.write(filedata) + +def _create_folder( + preset: dict[str, str], + cdn_folder: str, +): + if platform.system() == 'Linux' and not os.path.exists(os.path.join(cdn_folder, preset['out_folder'], 'chunks')): + os.makedirs(os.path.join(cdn_folder, preset['out_folder'], 'chunks')) + return + + # FFmpeg on Windows writes the chunk files + # to the chunks folder at the root of where ffmpeg is invoked + if not os.path.exists('chunks'): + os.makedirs('chunks') + + # The mpd file however is stored at the correct location + if not os.path.exists(os.path.join(cdn_folder, preset['out_folder'])): + os.makedirs(os.path.join(cdn_folder, preset['out_folder'])) + +def _encode( + preset: dict[str, str], + cdn_folder: str, + source_video: str, + aspect_ratio: str, + segment_duration: int = 10, + keyframe_interval: int = 2, +): + print(f"Encoding {preset['name']}") + + cmd = [ + "ffmpeg", "-v", "quiet", "-stats", + "-i", preset['input_video'], + "-i", source_video, + "-map", "0:v:0", # Video from Upscale + "-map", "1:a:0", # Audio from Source + "-c:v", preset['encoder'], + "-preset", preset['preset'], + "-crf", preset['crf'], + "-pix_fmt", "yuv420p", # 8bit to increase decode performance + "-vf", f"scale={preset['w']}:{preset['h']},setsar=1:1", + "-aspect", aspect_ratio, + ] + + if preset["encoder"] == "libx264": + cmd += ["-x264-params", f"keyint=24:min-keyint=24:scenecut=0"] + cmd += ["-c:a", "aac", "-b:a", "128k"] + elif preset["encoder"] == "libsvtav1": + cmd += ["-svtav1-params", f"keyint={keyframe_interval}s,fast-decode=1,tune=0"] + cmd += ["-c:a", "libopus", "-b:a", "128k"] + + cmd += [ + "-ac", "2", + "-sn", # No subtitles + "-map_metadata", "-1", # Get rid of metadata which might be incorrect + "-use_template", "1", # Don't list every segment url, use template instead + "-use_timeline", "1", # Make sure segment timing is always correct + "-init_seg_name", "chunks/init-stream$RepresentationID$.webm", # Init segment + "-media_seg_name", "chunks/chunk-stream$RepresentationID$-$Number%05d$.webm", # Media segments + "-seg_duration", str(segment_duration), # DASH segment duration + "-f", "dash", + os.path.join(cdn_folder, preset['out_folder'], 'manifest.mpd') + ] + + print(cmd) + + try: + subprocess.run(cmd, shell=True, check=True) + except subprocess.CalledProcessError as e: + print(f"\nffmpeg failed with error code {e.returncode}", file=sys.stderr) + sys.exit(e.returncode) + +def encode_streams( + cdn_folder: str, + source_video: str, + upscaled_video: str, + interpolated_video: str, + interpolated_uhd_video: str | None, + aspect_ratio: str = "16:9", +): + presets = [ + {"name": "720p", "w": "1280", "h": "720", "encoder": "libx264", "preset": "medium", "crf": "22", "input_video": upscaled_video, "out_folder": '720'}, + {"name": "1080p", "w": "1920", "h": "1080", "encoder": "libsvtav1", "preset": "6", "crf": "26", "input_video": upscaled_video, "out_folder": '1080'}, + {"name": "1080p48", "w": "1920", "h": "1080", "encoder": "libsvtav1", "preset": "6", "crf": "26", "input_video": interpolated_video, "out_folder": '1080i'}, + {"name": "2160", "w": "3840", "h": "2160", "encoder": "libsvtav1", "preset": "6", "crf": "28", "input_video": upscaled_video, "out_folder": '2160'}, + ] + + # Optional UHD Interpolate encode + if interpolated_uhd_video is not None: + presets.append({"name": "2160p48", "w": "3840", "h": "2160", "encoder": "libsvtav1", "preset": "6", "crf": "28", "input_video": interpolated_uhd_video, "out_folder": '2160i'}) + + for preset in presets: + # Skip already encoded streams + if os.path.exists(os.path.join(cdn_folder, preset['out_folder'], 'manifest.mpd')): + print(f"Skipped {preset['name']}") + continue + + _create_folder(preset, cdn_folder) + _encode(preset, cdn_folder, source_video, aspect_ratio) + _change_chunk_extension(preset, cdn_folder) + + _extract_subs(source_video, cdn_folder) + _encode_720p_fallback(cdn_folder, source_video, upscaled_video, aspect_ratio) + _create_sprites(cdn_folder) diff --git a/utils/interpolate.py b/utils/interpolate.py index e2bd01a..d746280 100644 --- a/utils/interpolate.py +++ b/utils/interpolate.py @@ -1,50 +1,81 @@ import os +import sys import subprocess -def createInterpolateScript(upscaleOut, tempName, inputAspect): - if os.path.isfile(upscaleOut + '.vpy'): - print('Interpolate script exists') - return - +def _create_vsrife_script( + vapoursynth_file: str, + hentai_name: str, + video_resolution: str = "1080p", + video_aspect: str = "16:9" +): script = [] - if inputAspect == '4:3': - script = ['from vsrife import rife', - 'import vapoursynth as vs', - 'from vapoursynth import core', - 'clip = core.ffms2.Source(source="./' + tempName + ' [4k][HEVC].mkv")', - 'clip = vs.core.resize.Bicubic(clip, width=1440, height=1080, format=vs.RGBS, matrix_in_s="709")', - 'clip = rife(clip=clip, model="4.25", factor_num=2, factor_den=1)', - 'clip = vs.core.resize.Bicubic(clip, format=vs.YUV420P8, matrix_s="709")', - 'clip.set_output()'] - else: - script = ['from vsrife import rife', - 'import vapoursynth as vs', - 'from vapoursynth import core', - 'clip = core.ffms2.Source(source="./' + tempName + ' [4k][HEVC].mkv")', - 'clip = vs.core.resize.Bicubic(clip, width=1920, height=1080, format=vs.RGBS, matrix_in_s="709")', - 'clip = rife(clip=clip, model="4.25", factor_num=2, factor_den=1)', - 'clip = vs.core.resize.Bicubic(clip, format=vs.YUV420P8, matrix_s="709")', - 'clip.set_output()'] + if video_resolution == '1080p': + video_width = "1440" if video_aspect == '4:3' else "1920" + script = [ + 'from vsrife import rife', + 'import vapoursynth as vs', + f'clip = vs.core.ffms2.Source(source="./{hentai_name} [4k][HEVC].mkv")', + f'clip = vs.core.resize.Bicubic(clip, width={video_width}, height=1080, format=vs.RGBS, matrix_in_s="709")', + 'clip = rife(clip=clip, model="4.25", factor_num=2, factor_den=1)', + 'clip = vs.core.resize.Bicubic(clip, format=vs.YUV420P8, matrix_s="709")', + 'clip.set_output()' + ] + elif video_resolution == '2160p': + video_width = "2880" if video_aspect == '4:3' else "3840" + script = [ + 'import vapoursynth as vs', + 'from vsrife import rife', + f'clip = vs.core.ffms2.Source(source="./{hentai_name} [4k][HEVC].mkv")', + f'clip = vs.core.resize.Bicubic(clip, width={video_width}, height=2160, format=vs.RGBS, matrix_in_s="709")', + 'clip = rife(clip=clip, model="4.25.lite", factor_num=2, factor_den=1)', + 'clip = vs.core.resize.Bicubic(clip, format=vs.YUV420P8, matrix_s="709")', + 'clip.set_output()' + ] - if not os.path.isfile(upscaleOut + '.vpy'): - with open(upscaleOut + '.vpy', 'a') as fs: - fs.writelines([i + '\n' for i in script]) + with open(vapoursynth_file, 'w') as fs: + fs.writelines([i + '\n' for i in script]) -def Interpolate(interpolateOut, upscaleOut, tempName, inputAspect): - if os.path.isfile(interpolateOut): +def _interpolate( + vapoursynth_file: str, + interpolate_output: str, +): + cmd = [ + "vspipe", + "-c", "y4m", + vapoursynth_file, + "-", "|", + "ffmpeg", "-v", "quiet", "-stats", + "-i", "-", + "-c:v", "hevc_nvenc", + "-qp", "5", + interpolate_output + ] + + try: + subprocess.run(cmd, shell=True, check=True) + except subprocess.CalledProcessError as e: + print(f"\nffmpeg failed with error code {e.returncode}", file=sys.stderr) + sys.exit(e.returncode) + +def interpolate( + upscale_video_input: str, + interpolate_output: str, + hentai_name: str, + video_resolution: str = "1080p", + video_aspect: str = "16:9", +): + if os.path.isfile(interpolate_output): print('Already interpolated') return - createInterpolateScript(upscaleOut, tempName, inputAspect) + vapoursynth_file = f"{upscale_video_input}.vpy" + ffindex_file = f"{upscale_video_input}.ffindex" - if not os.path.isfile(upscaleOut + '.vpy'): - print('=== Interpolation script not found ===') - return + _create_vsrife_script(vapoursynth_file, hentai_name, video_resolution, video_aspect) + _interpolate(vapoursynth_file, interpolate_output) - print('Interpolating') - subprocess.call('vspipe -c y4m "' + upscaleOut + '.vpy" - | ffmpeg -v quiet -stats -i - -c:v hevc_nvenc -qp 5 "' + interpolateOut + '"', shell=True) - - # Remove Temp Files - os.remove(upscaleOut + '.ffindex') - os.remove(upscaleOut + '.vpy') + # Cleanup + os.remove(ffindex_file) + os.remove(vapoursynth_file) + diff --git a/utils/interpolate4k.py b/utils/interpolate4k.py deleted file mode 100644 index 0a3071c..0000000 --- a/utils/interpolate4k.py +++ /dev/null @@ -1,42 +0,0 @@ -import os -import subprocess - -def createInterpolateScript(upscaleOut, tempName): - if os.path.isfile(upscaleOut + '.vpy'): - print('Interpolate script exists') - return - - script = ['from vsrife import rife', - 'import vapoursynth as vs', - 'from vapoursynth import core', - 'clip = core.ffms2.Source(source="./' + tempName + ' [4k][HEVC].mkv")', - 'clip = vs.core.resize.Bicubic(clip, width=3840, height=2160, format=vs.RGBS, matrix_in_s="709")', - 'clip = rife(clip=clip, model="4.25.lite", factor_num=2, factor_den=1)', - 'clip = vs.core.resize.Bicubic(clip, format=vs.YUV420P8, matrix_s="709")', - 'clip.set_output()'] - - if not os.path.isfile(upscaleOut + '.vpy'): - with open(upscaleOut + '.vpy', 'a') as fs: - fs.writelines([i + '\n' for i in script]) - -def Interpolate4K(interpolateOut, upscaleOut, interpolateVideo, tempName): - if not interpolateVideo: - print('Skipped interpolation') - return - - if os.path.isfile(interpolateOut): - print('Already interpolated') - return - - createInterpolateScript(upscaleOut, tempName) - - if not os.path.isfile(upscaleOut + '.vpy'): - print('=== Interpolation script not found ===') - return - - print('Interpolating') - subprocess.call('vspipe -c y4m "' + upscaleOut + '.vpy" - | ffmpeg -v quiet -stats -i - -c:v hevc_nvenc -qp 5 "' + interpolateOut + '"', shell=True) - - # Remove Temp Files - os.remove(upscaleOut + '.ffindex') - os.remove(upscaleOut + '.vpy') diff --git a/utils/mediainfo.py b/utils/mediainfo.py index 5608512..aa86746 100644 --- a/utils/mediainfo.py +++ b/utils/mediainfo.py @@ -1,30 +1,23 @@ from pymediainfo import MediaInfo -def get_aspect_ratio(inputFile): - media_info = MediaInfo.parse(inputFile) +def get_aspect_ratio(video_input: str) -> str: + media_info = MediaInfo.parse(video_input) + video_track = media_info.video_tracks[0] + aspect_ratio = video_track.other_display_aspect_ratio + print('Detected Aspect Ratio : ' + aspect_ratio[0]) + return aspect_ratio[0] - for track in media_info.tracks: - if track.track_type == "Video": - aspect_ratio = track.other_display_aspect_ratio - print('Detected Aspect Ratio : ' + aspect_ratio[0]) - return aspect_ratio[0] +def get_framerate(video_input: str) -> str: + media_info = MediaInfo.parse(video_input) + video_track = media_info.video_tracks[0] + frame_rate = video_track.frame_rate - # Fallback value - print('Falling Back To Aspect Ratio: 16:9') - return '16:9' - -def get_framerate(inputFile): - media_info = MediaInfo.parse(inputFile) - - for track in media_info.tracks: - if track.track_type == "Video": - frame_rate = track.frame_rate - if frame_rate == '29.970': - print(f"Detected Framerate : 30000/1001") - return '30000/1001' - if frame_rate == '24.000': - print(f"Detected Framerate : 24000/1000") - return '24000/1000' + if frame_rate == '29.970': + print(f"Detected Framerate : 30000/1001") + return '30000/1001' + if frame_rate == '24.000': + print(f"Detected Framerate : 24") + return '24' print(f"Detected Framerate : 24000/1001") return '24000/1001' \ No newline at end of file diff --git a/utils/upcale.py b/utils/upcale.py index 2fe6d7c..ffe449c 100644 --- a/utils/upcale.py +++ b/utils/upcale.py @@ -1,38 +1,92 @@ import os +import sys import subprocess from utils.mediainfo import get_framerate -def re_encode(input_file, upscale_out, temp_out, max_width, input_aspect): - if os.path.exists(upscale_out): - print('Skipped Pre-Encode') - return +MAX_INPUT_WIDTH = '720' + +def _re_encode( + source_video: str, + temp_out_video: str, + input_aspect: str = "16:9" +): + """ + Re-Encodes the source video to avoid nasty video bugs - command = ( - f'ffmpeg -v quiet -stats -i "{input_file}" ' - '-c:v ffv1 -level 3 ' - f'-vf "fps={get_framerate(input_file)},scale=-1:\'min({max_width},ih)\'" -aspect {input_aspect} ' - '-pix_fmt yuv420p -color_primaries 1 -color_trc 1 -colorspace 1 ' - '-an -sn -map_metadata -1 ' - f'"{temp_out}"' - ) + :param source_video: Video Input + :type source_video: str + :param input_aspect: Aspect Ratio of Video + :type input_aspect: str + """ - subprocess.call(command, shell=True) + cmd = [ + "ffmpeg", "-v", "quiet", "-stats", + "-i", source_video, + "-c:v", "ffv1", + "-level", "3", + "-vf", f"fps={get_framerate(source_video)},scale=-1:\'min({MAX_INPUT_WIDTH},ih)\'", + "-aspect", input_aspect, + "-pix_fmt", "yuv420p", + "-color_primaries", "1", + "-color_trc", "1", + "-colorspace", "1", + "-an", + "-sn", + "-map_metadata", "-1", + temp_out_video + ] -def upscale(input_file, upscale_out, max_width, input_aspect): - temp_out = os.path.join('1-Temp', 'source.mkv') - vsgan = os.path.join('utils', 'vs-realesrgan.vpy') + try: + subprocess.run(cmd, shell=True, check=True) + except subprocess.CalledProcessError as e: + print(f"\nffmpeg failed with error code {e.returncode} at _re_encode()", file=sys.stderr) + sys.exit(e.returncode) - # Re-Encode to fix issues - re_encode(input_file, upscale_out, temp_out, max_width, input_aspect) +def _upscale( + upscale_output: str, + input_aspect: str = "16:9" +): + print('Started Upscale') - if os.path.exists(upscale_out): + vapoursynth_script = os.path.join('utils', 'vs-realesrgan.vpy') + + cmd = [ + "vspipe", + "-c", "y4m", + vapoursynth_script, + "-", # Video output to pipe + "|", # Pipe + "ffmpeg", "-v", "quiet", "-stats", + "-f", "yuv4mpegpipe", + "-i", "-", # Pipe Video Input + "-c:v", "hevc_nvenc", + "-qp", "5", + "-aspect", input_aspect, + upscale_output + ] + + try: + subprocess.run(cmd, shell=True, check=True) + except subprocess.CalledProcessError as e: + print(f"\nffmpeg failed with error code {e.returncode}", file=sys.stderr) + sys.exit(e.returncode) + + +def upscale( + source_video: str, + upscaled_video_output: str, + input_aspect: str, +): + if os.path.exists(upscaled_video_output): print('Skipped Upscale') return + + temp_out_video = os.path.join('1-Temp', 'source.mkv') - print('Started Upscale') - subprocess.call(f'vspipe -c y4m {vsgan} - | ffmpeg -v quiet -stats -f yuv4mpegpipe -i - -c:v hevc_nvenc -qp 5 -aspect {input_aspect} "{upscale_out}"', shell=True) - + _re_encode(source_video, temp_out_video, input_aspect) + _upscale(upscaled_video_output, input_aspect) + # Remove Temp Files - os.remove(temp_out) - os.remove(temp_out + '.ffindex') + os.remove(temp_out_video) + os.remove(f'{temp_out_video}.ffindex')