{"@context":{"@vocab":"https://cir.nii.ac.jp/schema/1.0/","rdfs":"http://www.w3.org/2000/01/rdf-schema#","dc":"http://purl.org/dc/elements/1.1/","dcterms":"http://purl.org/dc/terms/","foaf":"http://xmlns.com/foaf/0.1/","prism":"http://prismstandard.org/namespaces/basic/2.0/","cinii":"http://ci.nii.ac.jp/ns/1.0/","datacite":"https://schema.datacite.org/meta/kernel-4/","ndl":"http://ndl.go.jp/dcndl/terms/","jpcoar":"https://github.com/JPCOAR/schema/blob/master/2.0/"},"@id":"https://cir.nii.ac.jp/crid/1360004235374172544.json","@type":"Article","productIdentifier":[{"identifier":{"@type":"DOI","@value":"10.1109/saner.2019.8668044"}},{"identifier":{"@type":"URI","@value":"http://xplorestaging.ieee.org/ielx7/8663526/8667965/08668044.pdf?arnumber=8668044"}}],"resourceType":"学術雑誌論文(journal article)","dc:title":[{"@value":"DeepCT: Tomographic Combinatorial Testing for Deep Learning Systems"}],"description":[{"notation":[{"@value":"Deep learning (DL) has achieved remarkable progress over the past decade and has been widely applied to many industry domains. However, the robustness of DL systems recently becomes great concerns, where minor perturbation on the input might cause the DL malfunction. These robustness issues could potentially result in severe consequences when a DL system is deployed to safety-critical applications and hinder the real-world deployment of DL systems. Testing techniques enable the robustness evaluation and vulnerable issue detection of a DL system at an early stage. The main challenge of testing a DL system attributes to the high dimensionality of its inputs and large internal latent feature space, which makes testing each state almost impossible. For traditional software, combinatorial testing (CT) is an effective testing technique to balance the testing exploration effort and defect detection capabilities. In this paper, we perform an exploratory study of CT on DL systems. We propose a set of combinatorial testing criteria specialized for DL systems, as well as a CT coverage guided test generation technique. Our evaluation demonstrates that CT provides a promising avenue for testing DL systems."}]}],"creator":[{"@id":"https://cir.nii.ac.jp/crid/1420001326213364992","@type":"Researcher","personIdentifier":[{"@type":"KAKEN_RESEARCHERS","@value":"70842061"},{"@type":"NRID","@value":"1000070842061"},{"@type":"NRID","@value":"9000241722210"},{"@type":"NRID","@value":"9000242084600"},{"@type":"RESEARCHMAP","@value":"https://researchmap.jp/malei"}],"foaf:name":[{"@value":"Lei Ma"}]},{"@id":"https://cir.nii.ac.jp/crid/1380004235374172544","@type":"Researcher","foaf:name":[{"@value":"Felix Juefei-Xu"}]},{"@id":"https://cir.nii.ac.jp/crid/1380004235374172675","@type":"Researcher","foaf:name":[{"@value":"Minhui Xue"}]},{"@id":"https://cir.nii.ac.jp/crid/1380004235374172545","@type":"Researcher","foaf:name":[{"@value":"Bo Li"}]},{"@id":"https://cir.nii.ac.jp/crid/1380004235374172672","@type":"Researcher","foaf:name":[{"@value":"Li Li"}]},{"@id":"https://cir.nii.ac.jp/crid/1380004235374172673","@type":"Researcher","foaf:name":[{"@value":"Yang Liu"}]},{"@id":"https://cir.nii.ac.jp/crid/1380004235374172674","@type":"Researcher","foaf:name":[{"@value":"Jianjun Zhao"}]}],"publication":{"prism:publicationName":[{"@value":"2019 IEEE 26th International Conference on Software Analysis, Evolution and Reengineering (SANER)"}],"dc:publisher":[{"@value":"IEEE"}],"prism:publicationDate":"2019-02","prism:startingPage":"614","prism:endingPage":"618"},"reviewed":"false","dc:rights":["https://ieeexplore.ieee.org/Xplorehelp/downloads/license-information/IEEE.html","https://doi.org/10.15223/policy-029","https://doi.org/10.15223/policy-037"],"url":[{"@id":"http://xplorestaging.ieee.org/ielx7/8663526/8667965/08668044.pdf?arnumber=8668044"}],"createdAt":"2019-03-19","modifiedAt":"2022-07-18","project":[{"@id":"https://cir.nii.ac.jp/crid/1040000781994306176","@type":"Project","projectIdentifier":[{"@type":"KAKEN","@value":"18H04097"},{"@type":"JGN","@value":"JP18H04097"},{"@type":"URI","@value":"https://kaken.nii.ac.jp/grant/KAKENHI-PROJECT-18H04097/"}],"notation":[{"@language":"ja","@value":"自動デバッグを可能にする群衆知エコシステムの確立"},{"@language":"en","@value":"Crowd Knowledge Ecosystems for Automatic Bug Fixing"}]}],"relatedProduct":[{"@id":"https://cir.nii.ac.jp/crid/1360013187865887744","@type":"Article","relationType":["references"],"jpcoar:relatedTitle":[{"@value":"Search-Based Software Testing: Past, Present and Future"}]},{"@id":"https://cir.nii.ac.jp/crid/1360292620436899456","@type":"Article","relationType":["references"],"jpcoar:relatedTitle":[{"@value":"DeepTest"}]},{"@id":"https://cir.nii.ac.jp/crid/1360294643815132544","@type":"Article","resourceType":"学術雑誌論文(journal article)","relationType":["isReferencedBy"],"jpcoar:relatedTitle":[{"@value":"<i>DeepRepair:</i> Style-Guided Repairing for Deep Neural Networks in the Real-World Operational Environment"}]},{"@id":"https://cir.nii.ac.jp/crid/1360567186237064576","@type":"Article","resourceType":"学術雑誌論文(journal article)","relationType":["references"],"jpcoar:relatedTitle":[{"@value":"DeepGauge: multi-granularity testing criteria for deep learning systems"}]},{"@id":"https://cir.nii.ac.jp/crid/1360574093497089280","@type":"Article","relationType":["references"],"jpcoar:relatedTitle":[{"@value":"The Building Blocks of Interpretability"}]},{"@id":"https://cir.nii.ac.jp/crid/1360574094009771776","@type":"Article","relationType":["references"],"jpcoar:relatedTitle":[{"@value":"Receptive fields, binocular interaction and functional architecture in the cat's visual cortex"}]},{"@id":"https://cir.nii.ac.jp/crid/1360576118790111872","@type":"Article","resourceType":"学術雑誌論文(journal article)","relationType":["isReferencedBy"],"jpcoar:relatedTitle":[{"@value":"NPC:\n            <u>N</u>\n            euron\n            <u>P</u>\n            ath\n            <u>C</u>\n            overage via Characterizing Decision Logic of Deep Neural Networks"}]},{"@id":"https://cir.nii.ac.jp/crid/1360848660085921280","@type":"Article","resourceType":"学術雑誌論文(journal article)","relationType":["references"],"jpcoar:relatedTitle":[{"@value":"GRT: Program-Analysis-Guided Random Testing (T)"}]},{"@id":"https://cir.nii.ac.jp/crid/1360861704768166400","@type":"Article","resourceType":"学術雑誌論文(journal article)","relationType":["isReferencedBy"],"jpcoar:relatedTitle":[{"@value":"FalsifAI: Falsification of AI-Enabled Hybrid Control Systems Guided by Time-Aware Coverage Criteria"}]},{"@id":"https://cir.nii.ac.jp/crid/1360861707142497792","@type":"Article","resourceType":"学術雑誌論文(journal article)","relationType":["isReferencedBy"],"jpcoar:relatedTitle":[{"@value":"PatchCensor: Patch Robustness Certification for Transformers via Exhaustive Testing"}]},{"@id":"https://cir.nii.ac.jp/crid/1360861707385003008","@type":"Article","resourceType":"学術雑誌論文(journal article)","relationType":["isReferencedBy"],"jpcoar:relatedTitle":[{"@value":"Machine learning application development: practitioners’ insights"}]},{"@id":"https://cir.nii.ac.jp/crid/1361694370282285824","@type":"Article","resourceType":"学術雑誌論文(journal article)","relationType":["isReferencedBy"],"jpcoar:relatedTitle":[{"@value":"Machine Learning Testing: Survey, Landscapes and Horizons"}]},{"@id":"https://cir.nii.ac.jp/crid/1362825893765446272","@type":"Article","relationType":["references"],"jpcoar:relatedTitle":[{"@value":"A survey of combinatorial testing"}]},{"@id":"https://cir.nii.ac.jp/crid/1363388845353771520","@type":"Article","relationType":["references"],"jpcoar:relatedTitle":[{"@value":"DeepXplore"}]},{"@id":"https://cir.nii.ac.jp/crid/1363388845760785280","@type":"Article","relationType":["references"],"jpcoar:relatedTitle":[{"@value":"Handling constraints in combinatorial interaction testing in the presence of multi objective particle swarm and multithreading"}]},{"@id":"https://cir.nii.ac.jp/crid/1363670321100913280","@type":"Article","relationType":["references"],"jpcoar:relatedTitle":[{"@value":"Optimization of Combinatorial Testing by Incremental SAT Solving"}]},{"@id":"https://cir.nii.ac.jp/crid/1370294662537622144","@type":"Product","relationType":["references"],"jpcoar:relatedTitle":[{"@value":"Deepcruiser: Automated guided testing for stateful deep learning systems"}]},{"@id":"https://cir.nii.ac.jp/crid/1370294662537622148","@type":"Product","relationType":["references"],"jpcoar:relatedTitle":[{"@value":"Verification for Machine Learning, Autonomy, and Neural Networks Survey"}]},{"@id":"https://cir.nii.ac.jp/crid/1370294662537622150","@type":"Product","relationType":["references"],"jpcoar:relatedTitle":[{"@value":"Reluplex: An efficient SMT solver for verifying deep neural networks"}]},{"@id":"https://cir.nii.ac.jp/crid/1370294662537622152","@type":"Product","relationType":["references"],"jpcoar:relatedTitle":[{"@value":"Learning combinatorial interaction test generation strategies using hyperheuristic search"}]},{"@id":"https://cir.nii.ac.jp/crid/1370294662537622154","@type":"Product","relationType":["references"],"jpcoar:relatedTitle":[{"@value":"DeepMutation: Mutation Testing of Deep Learning Systems"}]},{"@id":"https://cir.nii.ac.jp/crid/1370294662537622156","@type":"Product","relationType":["references"],"jpcoar:relatedTitle":[{"@value":"A survey of symbolic execution techniques"}]},{"@id":"https://cir.nii.ac.jp/crid/1370294662537622158","@type":"Product","relationType":["references"],"jpcoar:relatedTitle":[{"@value":"DeepHunter: Hunting Deep Neural Network Defects via Coverage-Guided Fuzzing"}]},{"@id":"https://cir.nii.ac.jp/crid/1370294662537622159","@type":"Product","relationType":["references"],"jpcoar:relatedTitle":[{"@value":"Explaining and harnessing adversarial examples"}]},{"@id":"https://cir.nii.ac.jp/crid/1370294662537622160","@type":"Product","relationType":["references"],"jpcoar:relatedTitle":[{"@value":"Pairwise testing in real world"}]},{"@id":"https://cir.nii.ac.jp/crid/1370294662537622161","@type":"Product","relationType":["references"],"jpcoar:relatedTitle":[{"@value":"Testing Deep Neural Networks"}]}],"dataSourceIdentifier":[{"@type":"CROSSREF","@value":"10.1109/saner.2019.8668044"},{"@type":"KAKEN","@value":"PRODUCT-22272757"},{"@type":"OPENAIRE","@value":"doi_dedup___::c36e85ade4d0df06e98dab6f44c94925"},{"@type":"CROSSREF","@value":"10.1109/tr.2021.3096332_references_DOI_1mQXeUnPC0XLy0gD2kuZy6Ia0tk"},{"@type":"CROSSREF","@value":"10.1109/tse.2019.2962027_references_DOI_1mQXeUnPC0XLy0gD2kuZy6Ia0tk"},{"@type":"CROSSREF","@value":"10.1145/3490489_references_DOI_1mQXeUnPC0XLy0gD2kuZy6Ia0tk"},{"@type":"CROSSREF","@value":"10.1109/tse.2022.3194640_references_DOI_1mQXeUnPC0XLy0gD2kuZy6Ia0tk"},{"@type":"CROSSREF","@value":"10.1145/3591870_references_DOI_1mQXeUnPC0XLy0gD2kuZy6Ia0tk"},{"@type":"CROSSREF","@value":"10.1007/s11219-023-09621-9_references_DOI_1mQXeUnPC0XLy0gD2kuZy6Ia0tk"}]}