294 lines
12 KiB
Python
294 lines
12 KiB
Python
# Copyright 2014 The Android Open Source Project
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import os
|
|
import os.path
|
|
import tempfile
|
|
import subprocess
|
|
import time
|
|
import sys
|
|
import textwrap
|
|
import its.device
|
|
from its.device import ItsSession
|
|
|
|
def main():
|
|
"""Run all the automated tests, saving intermediate files, and producing
|
|
a summary/report of the results.
|
|
|
|
Script should be run from the top-level CameraITS directory.
|
|
|
|
Command line Arguments:
|
|
camera: the camera(s) to be tested. Use comma to separate multiple
|
|
camera Ids. Ex: "camera=0,1" or "camera=1"
|
|
scenes: the test scene(s) to be executed. Use comma to separate multiple
|
|
scenes. Ex: "scenes=scene0,scene1" or "scenes=0,1,sensor_fusion"
|
|
(sceneX can be abbreviated by X where X is a integer)
|
|
chart: [Experimental] another android device served as test chart
|
|
display. When this argument presents, change of test scene will
|
|
be handled automatically. Note that this argument requires
|
|
special physical/hardware setup to work and may not work on
|
|
all android devices.
|
|
"""
|
|
|
|
SKIP_RET_CODE = 101
|
|
|
|
# Not yet mandated tests
|
|
NOT_YET_MANDATED = {
|
|
"scene0":[
|
|
"test_jitter"
|
|
],
|
|
"scene1":[
|
|
"test_ae_precapture_trigger",
|
|
"test_crop_region_raw",
|
|
"test_ev_compensation_advanced",
|
|
"test_ev_compensation_basic",
|
|
"test_yuv_plus_jpeg"
|
|
],
|
|
"scene2":[],
|
|
"scene3":[],
|
|
"scene4":[],
|
|
"scene5":[],
|
|
"sensor_fusion":[]
|
|
}
|
|
|
|
all_scenes = ["scene0", "scene1", "scene2", "scene3", "scene4", "scene5"]
|
|
|
|
auto_scenes = ["scene0", "scene1", "scene2", "scene3", "scene4"]
|
|
|
|
scene_req = {
|
|
"scene0" : None,
|
|
"scene1" : "A grey card covering at least the middle 30% of the scene",
|
|
"scene2" : "A picture containing human faces",
|
|
"scene3" : "A chart containing sharp edges like ISO 12233",
|
|
"scene4" : "A specific test page of a circle covering at least the "
|
|
"middle 50% of the scene. See CameraITS.pdf section 2.3.4 "
|
|
"for more details",
|
|
"scene5" : "Capture images with a diffuser attached to the camera. See "
|
|
"CameraITS.pdf section 2.3.4 for more details",
|
|
"sensor_fusion" : "Rotating checkboard pattern. See "
|
|
"sensor_fusion/SensorFusion.pdf for detailed "
|
|
"instructions. Note that this test will be skipped "
|
|
"on devices not supporting REALTIME camera timestamp."
|
|
"If that is the case, no scene setup is required and "
|
|
"you can just answer Y when being asked if the scene "
|
|
"is okay"
|
|
}
|
|
scene_extra_args = {
|
|
"scene5" : ["doAF=False"]
|
|
}
|
|
|
|
camera_ids = []
|
|
scenes = []
|
|
chart_host_id = None
|
|
for s in sys.argv[1:]:
|
|
if s[:7] == "camera=" and len(s) > 7:
|
|
camera_ids = s[7:].split(',')
|
|
elif s[:7] == "scenes=" and len(s) > 7:
|
|
scenes = s[7:].split(',')
|
|
elif s[:6] == 'chart=' and len(s) > 6:
|
|
chart_host_id = s[6:]
|
|
|
|
auto_scene_switch = chart_host_id is not None
|
|
|
|
# Run through all scenes if user does not supply one
|
|
possible_scenes = auto_scenes if auto_scene_switch else all_scenes
|
|
if not scenes:
|
|
scenes = possible_scenes
|
|
else:
|
|
# Validate user input scene names
|
|
valid_scenes = True
|
|
temp_scenes = []
|
|
for s in scenes:
|
|
if s in possible_scenes:
|
|
temp_scenes.append(s)
|
|
else:
|
|
try:
|
|
# Try replace "X" to "sceneX"
|
|
scene_num = int(s)
|
|
scene_str = "scene" + s
|
|
if scene_str not in possible_scenes:
|
|
valid_scenes = False
|
|
break
|
|
temp_scenes.append(scene_str)
|
|
except ValueError:
|
|
valid_scenes = False
|
|
break
|
|
|
|
if not valid_scenes:
|
|
print "Unknown scene specifiied:", s
|
|
assert(False)
|
|
scenes = temp_scenes
|
|
|
|
# Initialize test results
|
|
results = {}
|
|
result_key = ItsSession.RESULT_KEY
|
|
for s in all_scenes:
|
|
results[s] = {result_key: ItsSession.RESULT_NOT_EXECUTED}
|
|
|
|
# Make output directories to hold the generated files.
|
|
topdir = tempfile.mkdtemp()
|
|
print "Saving output files to:", topdir, "\n"
|
|
|
|
device_id = its.device.get_device_id()
|
|
device_id_arg = "device=" + device_id
|
|
print "Testing device " + device_id
|
|
|
|
# user doesn't specify camera id, run through all cameras
|
|
if not camera_ids:
|
|
camera_ids_path = os.path.join(topdir, "camera_ids.txt")
|
|
out_arg = "out=" + camera_ids_path
|
|
cmd = ['python',
|
|
os.path.join(os.getcwd(),"tools/get_camera_ids.py"), out_arg,
|
|
device_id_arg]
|
|
retcode = subprocess.call(cmd,cwd=topdir)
|
|
assert(retcode == 0)
|
|
with open(camera_ids_path, "r") as f:
|
|
for line in f:
|
|
camera_ids.append(line.replace('\n', ''))
|
|
|
|
print "Running ITS on camera: %s, scene %s" % (camera_ids, scenes)
|
|
|
|
if auto_scene_switch:
|
|
print 'Waking up chart screen: ', chart_host_id
|
|
screen_id_arg = ('screen=%s' % chart_host_id)
|
|
cmd = ['python', os.path.join(os.environ['CAMERA_ITS_TOP'], 'tools',
|
|
'wake_up_screen.py'), screen_id_arg]
|
|
retcode = subprocess.call(cmd)
|
|
assert retcode == 0
|
|
|
|
for camera_id in camera_ids:
|
|
# Loop capturing images until user confirm test scene is correct
|
|
camera_id_arg = "camera=" + camera_id
|
|
print "Preparing to run ITS on camera", camera_id
|
|
|
|
os.mkdir(os.path.join(topdir, camera_id))
|
|
for d in scenes:
|
|
os.mkdir(os.path.join(topdir, camera_id, d))
|
|
|
|
for scene in scenes:
|
|
tests = [(s[:-3],os.path.join("tests", scene, s))
|
|
for s in os.listdir(os.path.join("tests",scene))
|
|
if s[-3:] == ".py" and s[:4] == "test"]
|
|
tests.sort()
|
|
|
|
summary = "Cam" + camera_id + " " + scene + "\n"
|
|
numpass = 0
|
|
numskip = 0
|
|
num_not_mandated_fail = 0
|
|
numfail = 0
|
|
if scene_req[scene] != None:
|
|
out_path = os.path.join(topdir, camera_id, scene+".jpg")
|
|
out_arg = "out=" + out_path
|
|
if auto_scene_switch:
|
|
scene_arg = "scene=" + scene
|
|
cmd = ['python',
|
|
os.path.join(os.getcwd(), 'tools/load_scene.py'),
|
|
scene_arg, screen_id_arg]
|
|
else:
|
|
scene_arg = "scene=" + scene_req[scene]
|
|
extra_args = scene_extra_args.get(scene, [])
|
|
cmd = ['python',
|
|
os.path.join(os.getcwd(),"tools/validate_scene.py"),
|
|
camera_id_arg, out_arg,
|
|
scene_arg, device_id_arg] + extra_args
|
|
retcode = subprocess.call(cmd,cwd=topdir)
|
|
assert(retcode == 0)
|
|
print "Start running ITS on camera %s, %s" % (camera_id, scene)
|
|
|
|
# Run each test, capturing stdout and stderr.
|
|
for (testname,testpath) in tests:
|
|
if auto_scene_switch:
|
|
# Send an input event to keep the screen not dimmed.
|
|
# Since we are not using camera of chart screen, FOCUS event
|
|
# should does nothing but keep the screen from dimming.
|
|
# The "sleep after x minutes of inactivity" display setting
|
|
# determines how long this command can keep screen bright.
|
|
# Setting it to something like 30 minutes should be enough.
|
|
cmd = ('adb -s %s shell input keyevent FOCUS'
|
|
% chart_host_id)
|
|
subprocess.call(cmd.split())
|
|
cmd = ['python', os.path.join(os.getcwd(),testpath)] + \
|
|
sys.argv[1:] + [camera_id_arg]
|
|
outdir = os.path.join(topdir,camera_id,scene)
|
|
outpath = os.path.join(outdir,testname+"_stdout.txt")
|
|
errpath = os.path.join(outdir,testname+"_stderr.txt")
|
|
t0 = time.time()
|
|
with open(outpath,"w") as fout, open(errpath,"w") as ferr:
|
|
retcode = subprocess.call(
|
|
cmd,stderr=ferr,stdout=fout,cwd=outdir)
|
|
t1 = time.time()
|
|
|
|
test_failed = False
|
|
if retcode == 0:
|
|
retstr = "PASS "
|
|
numpass += 1
|
|
elif retcode == SKIP_RET_CODE:
|
|
retstr = "SKIP "
|
|
numskip += 1
|
|
elif retcode != 0 and testname in NOT_YET_MANDATED[scene]:
|
|
retstr = "FAIL*"
|
|
num_not_mandated_fail += 1
|
|
else:
|
|
retstr = "FAIL "
|
|
numfail += 1
|
|
test_failed = True
|
|
|
|
msg = "%s %s/%s [%.1fs]" % (retstr, scene, testname, t1-t0)
|
|
print msg
|
|
msg_short = "%s %s [%.1fs]" % (retstr, testname, t1-t0)
|
|
if test_failed:
|
|
summary += msg_short + "\n"
|
|
|
|
if numskip > 0:
|
|
skipstr = ", %d test%s skipped" % (
|
|
numskip, "s" if numskip > 1 else "")
|
|
else:
|
|
skipstr = ""
|
|
|
|
test_result = "\n%d / %d tests passed (%.1f%%)%s" % (
|
|
numpass + num_not_mandated_fail, len(tests) - numskip,
|
|
100.0 * float(numpass + num_not_mandated_fail) /
|
|
(len(tests) - numskip)
|
|
if len(tests) != numskip else 100.0,
|
|
skipstr)
|
|
print test_result
|
|
|
|
if num_not_mandated_fail > 0:
|
|
msg = "(*) tests are not yet mandated"
|
|
print msg
|
|
|
|
summary_path = os.path.join(topdir, camera_id, scene, "summary.txt")
|
|
with open(summary_path, "w") as f:
|
|
f.write(summary)
|
|
|
|
passed = numfail == 0
|
|
results[scene][result_key] = (ItsSession.RESULT_PASS if passed
|
|
else ItsSession.RESULT_FAIL)
|
|
results[scene][ItsSession.SUMMARY_KEY] = summary_path
|
|
|
|
print "Reporting ITS result to CtsVerifier"
|
|
its.device.report_result(device_id, camera_id, results)
|
|
|
|
if auto_scene_switch:
|
|
print 'Shutting down chart screen: ', chart_host_id
|
|
screen_id_arg = ('screen=%s' % chart_host_id)
|
|
cmd = ['python', os.path.join(os.environ['CAMERA_ITS_TOP'], 'tools',
|
|
'turn_off_screen.py'), screen_id_arg]
|
|
retcode = subprocess.call(cmd)
|
|
assert retcode == 0
|
|
|
|
print "ITS tests finished. Please go back to CtsVerifier and proceed"
|
|
|
|
if __name__ == '__main__':
|
|
main()
|