w33b/Upscale
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

merge into: w33b:main
Branches Tags
w33b:main w33b:linux-fixes
...
pull from: w33b:linux-fixes
Branches Tags
w33b:main w33b:linux-fixes

3 Commits
main ... linux-fixe

Author SHA1 Message Date
w33b
630ac6dd2d Fix running out of vram on linux 2026-03-05 23:07:44 +01:00
w33b
8fd85322ca Escape "$" on linux for DASH encode 2026-03-05 23:05:58 +01:00
w33b
15e19fa056 CMD list handled differently when shell=True 2026-03-05 23:03:59 +01:00
4 changed files with 333 additions and 309 deletions
Expand all files Collapse all files

13
utils/encode_downloads.py
View File

@@ -24,9 +24,9 @@ def _encode_video(
print(f'Encoding {preset['h']}p AV1') print(f'Encoding {preset['h']}p AV1')
cmd = [ cmd = [
"ffmpeg", "ffmpeg", "-v", "quiet", "-stats",
"-i", preset['input_video'], "-i", f'"{preset['input_video']}"',
"-i", source_video, "-i", f'"{source_video}"',
"-map", "0:v:0", # Video from upscale or interpolated file "-map", "0:v:0", # Video from upscale or interpolated file
"-map", "1:a:0", # Audio from source video "-map", "1:a:0", # Audio from source video
"-map", "1:s:0", # Subtitle from source video "-map", "1:s:0", # Subtitle from source video
@@ -38,15 +38,16 @@ def _encode_video(
"-crf", preset['crf'], "-crf", preset['crf'],
"-preset", "4", "-preset", "4",
"-pix_fmt", "yuv420p10le", # 10bit "-pix_fmt", "yuv420p10le", # 10bit
"-vf", f"scale=\'min({preset['w']},iw)\':-2,setsar=1:1", "-vf", f"\"scale=\'min({preset['w']},iw)\':-2,setsar=1:1\"",
"-aspect", input_aspect, "-aspect", input_aspect,
"-c:a", "libopus", "-c:a", "libopus",
"-b:a", "160k", "-b:a", "160k",
"-c:s", "copy", "-c:s", "copy",
output_video f'"{output_video}"'
] ]
print(cmd) if sys.platform == 'linux':
cmd = ' '.join(cmd)
try: try:
subprocess.run(cmd, shell=True, check=True) subprocess.run(cmd, shell=True, check=True)

429
utils/encode_stream.py
View File

@@ -1,209 +1,220 @@
import os import os
import subprocess import subprocess
import shutil import shutil
import platform import platform
import sys import sys
def _extract_subs( def _extract_subs(
source_video: str, source_video: str,
cdn_folder: str cdn_folder: str
): ):
out_ass = os.path.join(cdn_folder, 'eng.ass') out_ass = os.path.join(cdn_folder, 'eng.ass')
out_vtt = os.path.join(cdn_folder, 'eng.vtt') out_vtt = os.path.join(cdn_folder, 'eng.vtt')
if os.path.exists(out_ass): if os.path.exists(out_ass):
print('Skipped Sub Extract') print('Skipped Sub Extract')
return return
print('Extracting Sub') print('Extracting Sub')
subprocess.call(f'ffmpeg -y -v quiet -stats -i "{source_video}" -c copy "{out_ass}"', shell=True) subprocess.call(f'ffmpeg -y -v quiet -stats -i "{source_video}" -c copy "{out_ass}"', shell=True)
subprocess.call(f'ffmpeg -y -v quiet -stats -i "{out_ass}" "{out_vtt}"', shell=True) subprocess.call(f'ffmpeg -y -v quiet -stats -i "{out_ass}" "{out_vtt}"', shell=True)
def _create_sprites(cdn_folder: str): def _create_sprites(cdn_folder: str):
""" """
Creates video player sprites Creates video player sprites
""" """
video_file = os.path.join(cdn_folder, 'x264.720p.mp4') video_file = os.path.join(cdn_folder, 'x264.720p.mp4')
# Generating Sprites # Generating Sprites
if not os.path.exists(os.path.join(cdn_folder, 'thumbs.vtt')) and os.path.exists(video_file): if not os.path.exists(os.path.join(cdn_folder, 'thumbs.vtt')) and os.path.exists(video_file):
os.system(f'python makesprites.py "{video_file}"') os.system(f'python makesprites.py "{video_file}"')
os.rename("thumbs.vtt", os.path.join(cdn_folder, 'thumbs.vtt')) os.rename("thumbs.vtt", os.path.join(cdn_folder, 'thumbs.vtt'))
os.rename("sprite.jpg", os.path.join(cdn_folder, 'sprite.jpg')) os.rename("sprite.jpg", os.path.join(cdn_folder, 'sprite.jpg'))
shutil.rmtree('thumbs') shutil.rmtree('thumbs')
shutil.rmtree('logs') shutil.rmtree('logs')
return return
print('Skipped Sprites') print('Skipped Sprites')
def _encode_720p_fallback( def _encode_720p_fallback(
cdn_folder: str, cdn_folder: str,
video_source: str, video_source: str,
upscale_output: str, upscale_output: str,
aspect_ratio: str = "16:9" aspect_ratio: str = "16:9"
): ):
""" """
Fallback video stream for apple devices Fallback video stream for apple devices
""" """
output = os.path.join(cdn_folder, 'x264.720p.mp4') output = os.path.join(cdn_folder, 'x264.720p.mp4')
if os.path.exists(output): if os.path.exists(output):
print('Skipped 720p Encode') print('Skipped 720p Encode')
return return
cmd = [ cmd = [
"ffmpeg", "-v", "quiet", "-stats", "ffmpeg", "-v", "quiet", "-stats",
"-i", upscale_output, "-i", f'"{upscale_output}"',
"-i", video_source, "-i", f'"{video_source}"',
"-map", "0:v:0", "-map", "0:v:0",
"-map", "1:a:0", "-map", "1:a:0",
"-c:v", "libx264", "-c:v", "libx264",
"-pix_fmt", "yuv420p", "-pix_fmt", "yuv420p",
"-vf", "scale=1280:720,setsar=1:1", "-vf", "scale=1280:720,setsar=1:1",
"-aspect", aspect_ratio, "-aspect", aspect_ratio,
"-c:a", "aac", "-c:a", "aac",
"-b:a", "128k", "-b:a", "128k",
"-sn", "-sn",
"-map_metadata", "-1", "-map_metadata", "-1",
"-movflags", "+faststart", "-movflags", "+faststart",
output f'"{output}"'
] ]
try: if sys.platform == 'linux':
subprocess.run(cmd, shell=True, check=True) cmd = ' '.join(cmd)
except subprocess.CalledProcessError as e:
print(f"\nffmpeg failed with error code {e.returncode}", file=sys.stderr) try:
sys.exit(e.returncode) subprocess.run(cmd, shell=True, check=True)
except subprocess.CalledProcessError as e:
print(f"\nffmpeg failed with error code {e.returncode}", file=sys.stderr)
def _change_chunk_extension( sys.exit(e.returncode)
preset: dict[str, str],
cdn_folder: str,
): def _change_chunk_extension(
chunks_folder = os.path.join(cdn_folder, preset['out_folder'], 'chunks') preset: dict[str, str],
mpd_file = os.path.join(cdn_folder, preset['out_folder'], 'manifest.mpd') cdn_folder: str,
):
# Move encodes on Windows to correct folder chunks_folder = os.path.join(cdn_folder, preset['out_folder'], 'chunks')
if platform.system() != 'Linux': mpd_file = os.path.join(cdn_folder, preset['out_folder'], 'manifest.mpd')
shutil.move('chunks', chunks_folder)
# Move encodes on Windows to correct folder
# Rename files if platform.system() != 'Linux':
for filename in os.listdir(chunks_folder): shutil.move('chunks', chunks_folder)
file_path = os.path.join(chunks_folder, filename)
if not os.path.isfile(file_path): # Rename files
continue for filename in os.listdir(chunks_folder):
file_path = os.path.join(chunks_folder, filename)
new_file_path = file_path.replace('.webm', '.webp') if not os.path.isfile(file_path):
os.rename(file_path, new_file_path) continue
# Modify manifest new_file_path = file_path.replace('.webm', '.webp')
with open(mpd_file, 'r') as file : os.rename(file_path, new_file_path)
filedata = file.read()
# Modify manifest
# Replace the target string with open(mpd_file, 'r') as file :
filedata = filedata.replace('.webm', '.webp') filedata = file.read()
# Write the file out again # Replace the target string
with open(mpd_file, 'w') as file: filedata = filedata.replace('.webm', '.webp')
file.write(filedata)
# Write the file out again
def _create_folder( with open(mpd_file, 'w') as file:
preset: dict[str, str], file.write(filedata)
cdn_folder: str,
): def _create_folder(
if platform.system() == 'Linux' and not os.path.exists(os.path.join(cdn_folder, preset['out_folder'], 'chunks')): preset: dict[str, str],
os.makedirs(os.path.join(cdn_folder, preset['out_folder'], 'chunks')) cdn_folder: str,
return ):
if platform.system() == 'Linux' and not os.path.exists(os.path.join(cdn_folder, preset['out_folder'], 'chunks')):
# FFmpeg on Windows writes the chunk files os.makedirs(os.path.join(cdn_folder, preset['out_folder'], 'chunks'))
# to the chunks folder at the root of where ffmpeg is invoked return
if not os.path.exists('chunks'):
os.makedirs('chunks') # FFmpeg on Windows writes the chunk files
# to the chunks folder at the root of where ffmpeg is invoked
# The mpd file however is stored at the correct location if not os.path.exists('chunks'):
if not os.path.exists(os.path.join(cdn_folder, preset['out_folder'])): os.makedirs('chunks')
os.makedirs(os.path.join(cdn_folder, preset['out_folder']))
# The mpd file however is stored at the correct location
def _encode( if not os.path.exists(os.path.join(cdn_folder, preset['out_folder'])):
preset: dict[str, str], os.makedirs(os.path.join(cdn_folder, preset['out_folder']))
cdn_folder: str,
source_video: str, def _encode(
aspect_ratio: str, preset: dict[str, str],
segment_duration: int = 10, cdn_folder: str,
keyframe_interval: int = 2, source_video: str,
): aspect_ratio: str,
print(f"Encoding {preset['name']}") segment_duration: int = 10,
keyframe_interval: int = 2,
cmd = [ ):
"ffmpeg", "-v", "quiet", "-stats", print(f"Encoding {preset['name']}")
"-i", preset['input_video'],
"-i", source_video, cmd = [
"-map", "0:v:0", # Video from Upscale "ffmpeg", "-v", "quiet", "-stats",
"-map", "1:a:0", # Audio from Source "-i", f'"{preset['input_video']}"',
"-c:v", preset['encoder'], "-i", f'"{source_video}"',
"-preset", preset['preset'], "-map", "0:v:0", # Video from Upscale
"-crf", preset['crf'], "-map", "1:a:0", # Audio from Source
"-pix_fmt", "yuv420p", # 8bit to increase decode performance "-c:v", preset['encoder'],
"-vf", f"scale={preset['w']}:{preset['h']},setsar=1:1", "-preset", preset['preset'],
"-aspect", aspect_ratio, "-crf", preset['crf'],
] "-pix_fmt", "yuv420p", # 8bit to increase decode performance
"-vf", f"scale={preset['w']}:{preset['h']},setsar=1:1",
if preset["encoder"] == "libx264": "-aspect", aspect_ratio,
cmd += ["-x264-params", f"keyint=24:min-keyint=24:scenecut=0"] ]
cmd += ["-c:a", "aac", "-b:a", "128k"]
elif preset["encoder"] == "libsvtav1": if preset["encoder"] == "libx264":
cmd += ["-svtav1-params", f"keyint={keyframe_interval}s,fast-decode=1,tune=0"] cmd += ["-x264-params", f"keyint=24:min-keyint=24:scenecut=0"]
cmd += ["-c:a", "libopus", "-b:a", "128k"] cmd += ["-c:a", "aac", "-b:a", "128k"]
elif preset["encoder"] == "libsvtav1":
cmd += [ cmd += ["-svtav1-params", f"keyint={keyframe_interval}s,fast-decode=1,tune=0"]
"-ac", "2", cmd += ["-c:a", "libopus", "-b:a", "128k"]
"-sn", # No subtitles
"-map_metadata", "-1", # Get rid of metadata which might be incorrect init_seg_name = "chunks/init-stream$RepresentationID$.webm"
"-use_template", "1", # Don't list every segment url, use template instead media_seg_name = "chunks/chunk-stream$RepresentationID$-$Number%05d$.webm"
"-use_timeline", "1", # Make sure segment timing is always correct
"-init_seg_name", "chunks/init-stream$RepresentationID$.webm", # Init segment if sys.platform == 'linux':
"-media_seg_name", "chunks/chunk-stream$RepresentationID$-$Number%05d$.webm", # Media segments init_seg_name = "chunks/init-stream\$RepresentationID\$.webm"
"-seg_duration", str(segment_duration), # DASH segment duration media_seg_name = "chunks/chunk-stream\$RepresentationID\$-\$Number%05d\$.webm"
"-f", "dash",
os.path.join(cdn_folder, preset['out_folder'], 'manifest.mpd') cmd += [
] "-ac", "2",
"-sn", # No subtitles
print(cmd) "-map_metadata", "-1", # Get rid of metadata which might be incorrect
"-use_template", "1", # Don't list every segment url, use template instead
try: "-use_timeline", "1", # Make sure segment timing is always correct
subprocess.run(cmd, shell=True, check=True) "-init_seg_name", init_seg_name, # Init segment
except subprocess.CalledProcessError as e: "-media_seg_name", media_seg_name, # Media segments
print(f"\nffmpeg failed with error code {e.returncode}", file=sys.stderr) "-seg_duration", str(segment_duration), # DASH segment duration
sys.exit(e.returncode) "-f", "dash",
f'"{os.path.join(cdn_folder, preset['out_folder'], 'manifest.mpd')}"'
def encode_streams( ]
cdn_folder: str,
source_video: str, if sys.platform == 'linux':
upscaled_video: str, cmd = ' '.join(cmd)
interpolated_video: str,
interpolated_uhd_video: str | None, try:
aspect_ratio: str = "16:9", subprocess.run(cmd, shell=True, check=True)
): except subprocess.CalledProcessError as e:
presets = [ print(f"\nffmpeg failed with error code {e.returncode}", file=sys.stderr)
{"name": "720p", "w": "1280", "h": "720", "encoder": "libx264", "preset": "medium", "crf": "22", "input_video": upscaled_video, "out_folder": '720'}, sys.exit(e.returncode)
{"name": "1080p", "w": "1920", "h": "1080", "encoder": "libsvtav1", "preset": "6", "crf": "26", "input_video": upscaled_video, "out_folder": '1080'},
{"name": "1080p48", "w": "1920", "h": "1080", "encoder": "libsvtav1", "preset": "6", "crf": "26", "input_video": interpolated_video, "out_folder": '1080i'}, def encode_streams(
{"name": "2160", "w": "3840", "h": "2160", "encoder": "libsvtav1", "preset": "6", "crf": "28", "input_video": upscaled_video, "out_folder": '2160'}, cdn_folder: str,
] source_video: str,
upscaled_video: str,
# Optional UHD Interpolate encode interpolated_video: str,
if interpolated_uhd_video is not None: interpolated_uhd_video: str | None,
presets.append({"name": "2160p48", "w": "3840", "h": "2160", "encoder": "libsvtav1", "preset": "6", "crf": "28", "input_video": interpolated_uhd_video, "out_folder": '2160i'}) aspect_ratio: str = "16:9",
):
for preset in presets: presets = [
# Skip already encoded streams {"name": "720p", "w": "1280", "h": "720", "encoder": "libx264", "preset": "medium", "crf": "22", "input_video": upscaled_video, "out_folder": '720'},
if os.path.exists(os.path.join(cdn_folder, preset['out_folder'], 'manifest.mpd')): {"name": "1080p", "w": "1920", "h": "1080", "encoder": "libsvtav1", "preset": "6", "crf": "26", "input_video": upscaled_video, "out_folder": '1080'},
print(f"Skipped {preset['name']}") {"name": "1080p48", "w": "1920", "h": "1080", "encoder": "libsvtav1", "preset": "6", "crf": "26", "input_video": interpolated_video, "out_folder": '1080i'},
continue {"name": "2160", "w": "3840", "h": "2160", "encoder": "libsvtav1", "preset": "6", "crf": "28", "input_video": upscaled_video, "out_folder": '2160'},
]
_create_folder(preset, cdn_folder)
_encode(preset, cdn_folder, source_video, aspect_ratio) # Optional UHD Interpolate encode
_change_chunk_extension(preset, cdn_folder) if interpolated_uhd_video is not None:
presets.append({"name": "2160p48", "w": "3840", "h": "2160", "encoder": "libsvtav1", "preset": "6", "crf": "28", "input_video": interpolated_uhd_video, "out_folder": '2160i'})
_extract_subs(source_video, cdn_folder)
_encode_720p_fallback(cdn_folder, source_video, upscaled_video, aspect_ratio) for preset in presets:
_create_sprites(cdn_folder) # Skip already encoded streams
if os.path.exists(os.path.join(cdn_folder, preset['out_folder'], 'manifest.mpd')):
print(f"Skipped {preset['name']}")
continue
_create_folder(preset, cdn_folder)
_encode(preset, cdn_folder, source_video, aspect_ratio)
_change_chunk_extension(preset, cdn_folder)
_extract_subs(source_video, cdn_folder)
_encode_720p_fallback(cdn_folder, source_video, upscaled_video, aspect_ratio)
_create_sprites(cdn_folder)

10
utils/interpolate.py
View File

@@ -26,6 +26,9 @@ def _create_vsrife_script(
script = [ script = [
'import vapoursynth as vs', 'import vapoursynth as vs',
'from vsrife import rife', 'from vsrife import rife',
# this isn't a real fix, as this SHOULDN'T FIX IT
# however for some reason it does
'vs.core.max_cache_size=8192',
f'clip = vs.core.ffms2.Source(source="./{hentai_name} [4k][HEVC].mkv")', f'clip = vs.core.ffms2.Source(source="./{hentai_name} [4k][HEVC].mkv")',
f'clip = vs.core.resize.Bicubic(clip, width={video_width}, height=2160, format=vs.RGBS, matrix_in_s="709")', f'clip = vs.core.resize.Bicubic(clip, width={video_width}, height=2160, format=vs.RGBS, matrix_in_s="709")',
'clip = rife(clip=clip, model="4.25.lite", factor_num=2, factor_den=1)', 'clip = rife(clip=clip, model="4.25.lite", factor_num=2, factor_den=1)',
@@ -43,15 +46,18 @@ def _interpolate(
cmd = [ cmd = [
"vspipe", "vspipe",
"-c", "y4m", "-c", "y4m",
vapoursynth_file, f'"{vapoursynth_file}"',
"-", "|", "-", "|",
"ffmpeg", "-v", "quiet", "-stats", "ffmpeg", "-v", "quiet", "-stats",
"-i", "-", "-i", "-",
"-c:v", "hevc_nvenc", "-c:v", "hevc_nvenc",
"-qp", "5", "-qp", "5",
interpolate_output f'"{interpolate_output}"'
] ]
if sys.platform == 'linux':
cmd = ' '.join(cmd)
try: try:
subprocess.run(cmd, shell=True, check=True) subprocess.run(cmd, shell=True, check=True)
except subprocess.CalledProcessError as e: except subprocess.CalledProcessError as e:

190
utils/upcale.py
View File

@@ -1,92 +1,98 @@
import os import os
import sys import sys
import subprocess import subprocess
from utils.mediainfo import get_framerate from utils.mediainfo import get_framerate
MAX_INPUT_WIDTH = '720' MAX_INPUT_WIDTH = '720'
def _re_encode( def _re_encode(
source_video: str, source_video: str,
temp_out_video: str, temp_out_video: str,
input_aspect: str = "16:9" input_aspect: str = "16:9"
): ):
""" """
Re-Encodes the source video to avoid nasty video bugs Re-Encodes the source video to avoid nasty video bugs
:param source_video: Video Input :param source_video: Video Input
:type source_video: str :type source_video: str
:param input_aspect: Aspect Ratio of Video :param input_aspect: Aspect Ratio of Video
:type input_aspect: str :type input_aspect: str
""" """
cmd = [ cmd = [
"ffmpeg", "-v", "quiet", "-stats", "ffmpeg", "-v", "quiet", "-stats",
"-i", source_video, "-i", f'"{source_video}"',
"-c:v", "ffv1", "-c:v", "ffv1",
"-level", "3", "-level", "3",
"-vf", f"fps={get_framerate(source_video)},scale=-1:\'min({MAX_INPUT_WIDTH},ih)\'", "-vf", f"\"fps={get_framerate(source_video)},scale=-1:\'min({MAX_INPUT_WIDTH},ih)\'\"",
"-aspect", input_aspect, "-aspect", input_aspect,
"-pix_fmt", "yuv420p", "-pix_fmt", "yuv420p",
"-color_primaries", "1", "-color_primaries", "1",
"-color_trc", "1", "-color_trc", "1",
"-colorspace", "1", "-colorspace", "1",
"-an", "-an",
"-sn", "-sn",
"-map_metadata", "-1", "-map_metadata", "-1",
temp_out_video f'"{temp_out_video}"'
] ]
try: if sys.platform == 'linux':
subprocess.run(cmd, shell=True, check=True) cmd = ' '.join(cmd)
except subprocess.CalledProcessError as e:
print(f"\nffmpeg failed with error code {e.returncode} at _re_encode()", file=sys.stderr) try:
sys.exit(e.returncode) subprocess.run(cmd, shell=True, check=True)
except subprocess.CalledProcessError as e:
def _upscale( print(f"\nffmpeg failed with error code {e.returncode} at _re_encode()", file=sys.stderr)
upscale_output: str, sys.exit(e.returncode)
input_aspect: str = "16:9"
): def _upscale(
print('Started Upscale') upscale_output: str,
input_aspect: str = "16:9"
vapoursynth_script = os.path.join('utils', 'vs-realesrgan.vpy') ):
print('Started Upscale')
cmd = [
"vspipe", vapoursynth_script = os.path.join('utils', 'vs-realesrgan.vpy')
"-c", "y4m",
vapoursynth_script, cmd = [
"-", # Video output to pipe "vspipe",
"|", # Pipe "-c", "y4m",
"ffmpeg", "-v", "quiet", "-stats", f"\"{vapoursynth_script}\"",
"-f", "yuv4mpegpipe", "-", # Video output to pipe
"-i", "-", # Pipe Video Input "|", # Pipe
"-c:v", "hevc_nvenc", "ffmpeg", "-v", "quiet", "-stats",
"-qp", "5", "-f", "yuv4mpegpipe",
"-aspect", input_aspect, "-i", "-", # Pipe Video Input
upscale_output "-c:v", "hevc_nvenc",
] "-qp", "5",
"-aspect", input_aspect,
try: f"\"{upscale_output}\""
subprocess.run(cmd, shell=True, check=True) ]
except subprocess.CalledProcessError as e:
print(f"\nffmpeg failed with error code {e.returncode}", file=sys.stderr) if sys.platform == 'linux':
sys.exit(e.returncode) cmd = ' '.join(cmd)
try:
def upscale( subprocess.run(cmd, shell=True, check=True)
source_video: str, except subprocess.CalledProcessError as e:
upscaled_video_output: str, print(f"\nffmpeg failed with error code {e.returncode}", file=sys.stderr)
input_aspect: str, sys.exit(e.returncode)
):
if os.path.exists(upscaled_video_output):
print('Skipped Upscale') def upscale(
return source_video: str,
upscaled_video_output: str,
temp_out_video = os.path.join('1-Temp', 'source.mkv') input_aspect: str,
):
_re_encode(source_video, temp_out_video, input_aspect) if os.path.exists(upscaled_video_output):
_upscale(upscaled_video_output, input_aspect) print('Skipped Upscale')
return
# Remove Temp Files
os.remove(temp_out_video) temp_out_video = os.path.join('1-Temp', 'source.mkv')
os.remove(f'{temp_out_video}.ffindex')
_re_encode(source_video, temp_out_video, input_aspect)
_upscale(upscaled_video_output, input_aspect)
# Remove Temp Files
os.remove(temp_out_video)
os.remove(f'{temp_out_video}.ffindex')