Azure OpenAI Sora: Video-to-Video Inpainting Validation Errors

Hi Su Myat Hlaing

I found this schema in the below docs.

data = {
    "prompt": "A serene forest scene transitioning into autumn",
    "height": str(1080),
    "width": str(1920),
    "n_seconds": str(10),
    "n_variants": str(1),
    "model": "sora",
    # inpaint_items must be JSON string
    "inpaint_items": json.dumps([
        {
            "frame_index": 0,
            "type": "video",
            "file_name": "dog_swimming.mp4",
            "crop_bounds": {
                "left_fraction": 0.1,
                "top_fraction": 0.1,
                "right_fraction": 0.9,
                "bottom_fraction": 0.9
            }
        }
    ])
}

https://learn.microsoft.com/en-us/azure/ai-foundry/openai/video-generation-quickstart?tabs=windows%2Ckeyless%2Cvideo-prompt&pivots=rest-api#generate-video-with-sora

Please clarify more with a sample SDK/API call with respect to start_frame_index and end_frame_index

Thank you.

@Manas Mohanty Thank you for the example. However, I am already using the exact same schema from the documentation that you provided and occurred below error.

from dotenv import load_dotenv
import os
import time
import requests
import json
from datetime import datetime

# Load environment variables from .env file
load_dotenv()

# --- Configuration ---
ENDPOINT = os.getenv("AZURE_OPENAI_ENDPOINT")
API_KEY = os.getenv("AZURE_OPENAI_API_KEY")

# The script assumes a file named 'neon.mp4' exists in the current directory.
VIDEO_FILE_NAME = "neon.mp4"

# Set up headers with the API key
headers = {
    "api-key": API_KEY,
}

# --- Utility Check ---
def check_prerequisites():
    if not ENDPOINT or not API_KEY:
        print("❌ Please set AZURE_OPENAI_ENDPOINT and AZURE_OPENAI_API_KEY in .env file.")
        return False
    if not os.path.exists(VIDEO_FILE_NAME):
        print(f"❌ Input video file not found: {VIDEO_FILE_NAME}. Create a file with this name for testing.")
        return False
    return True

if __name__ == "__main__":
    
    if not check_prerequisites():
        exit()

    try:
        # 1. Create a video generation job with video inpainting (multipart upload)
        create_url = f"{ENDPOINT}/openai/v1/video/generations/jobs?api-version=preview"
        
        # Flatten the body for multipart/form-data
        data = {
            "prompt": "change to animation style",
            "height": str(1080),
            "width": str(1920),
            "n_seconds": str(10),
            "n_variants": str(1),
            "model": "sora",
            # inpaint_items must be JSON string
            "inpaint_items": json.dumps([
                {
                    "frame_index": 0,
                    "type": "video",
                    "file_name": VIDEO_FILE_NAME,
                    "crop_bounds": {
                        "left_fraction": 0.1,
                        "top_fraction": 0.1,
                        "right_fraction": 0.9,
                        "bottom_fraction": 0.9
                    }
                }
            ])
        }
        
        # Replace with your own video file path
        print(f"\n--- 1. Creating Job for {VIDEO_FILE_NAME} ---")
        with open(VIDEO_FILE_NAME, "rb") as video_file:
            files = [
                ("files", (VIDEO_FILE_NAME, video_file, "video/mp4"))
            ]
            
            multipart_headers = {k: v for k, v in headers.items() if k.lower() != "content-type"}
            
            response = requests.post(
                create_url,
                headers=multipart_headers,
                data=data,
                files=files
            )

        if not response.ok:
            print("❌ Error response:", response.status_code, response.text)
            response.raise_for_status()
        
        job_data = response.json()
        print("Full response JSON:", job_data)
        job_id = job_data["id"]
        print(f"✅ Job created: {job_id}")

        # 2. Poll for job status
        print(f"\n--- 2. Polling Status for Job ID: {job_id} ---")
        status_url = f"{ENDPOINT}/openai/v1/video/generations/jobs/{job_id}?api-version=preview"
        status = None
        status_response = {}
        
        while status not in ("succeeded", "failed", "cancelled"):
            time.sleep(5)
            status_response = requests.get(status_url, headers=headers).json()
            status = status_response.get("status")
            print(f"[{datetime.now().strftime('%H:%M:%S')}] Job status: {status}")

        # 3. Retrieve generated video
        if status == "succeeded":
            print("\n--- 3. Job Succeeded. Retrieving Video ---")
            generations = status_response.get("generations", [])
            if generations:
                generation_id = generations[0].get("id")
                video_url = f"{ENDPOINT}/openai/v1/video/generations/{generation_id}/content/video?api-version=preview"
                print(f"Attempting to download video for Generation ID: {generation_id}")
                
                video_response = requests.get(video_url, headers=headers)
                
                if video_response.ok:
                    output_filename = "output_generated.mp4"
                    with open(output_filename, "wb") as file:
                        file.write(video_response.content)
                        print(f'✅ Generated video saved as "{output_filename}"')
                else:
                    print(f"❌ Failed to download video: {video_response.status_code} - {video_response.text}")
                    raise Exception("Failed to retrieve generated video content.")
            else:
                raise Exception("No generations found in job result.")
        else:
            reason = status_response.get("failure_reason", "Unknown reason.")
            raise Exception(f"❌ Job didn't succeed. Final Status: {status}. Reason: {reason}")
            
    except Exception as e:
        print(f"\nAn exception occurred: {e}")
        
    print("\nProcess finished.")

Su Myat Hlaing

Thank you for providing more clarity. I am able to replicate the issue at my side and syncing with product group internally on this.

Shall update you soon.

Su Myat Hlaing

I am still waiting for working code with example input from product group.

Shall update you once we progress.

Thank you.

@Manas Mohanty Thank you for the update. I appreciate your follow-up and will wait for the product group’s response.