This shell script finds duplicate files in a given directory comparing their (md5) checksum. This means the content is checked and is strictly identical, rather than the filename or date of creation.
\nThis is usually useful to delete large files. ‘Find’ command option -size can help speeding up and finding the largest duplicate files.
\n <\/p>\n
admin@fileserver$\nfind \/usr\/bin -type f -print0 |\nxargs -0 -n1 md5sum |\nsort -k 1,32 |\nuniq -w 32 -d --all-repeated=separate |\nsed -e 's\/^[0-9a-f]*\\ *\/\/;'\n\n\/usr\/bin\/c2ph\n\/usr\/bin\/pstruct\n\n\/usr\/bin\/pgrep\n\/usr\/bin\/pkill\n\n\/usr\/bin\/perl\n\/usr\/bin\/perl5.8.8\n\/usr\/bin\/suidperl\n...\n<\/pre>\nThis could be run on Windows file systems mounted via Samba.<\/p>\n","protected":false},"excerpt":{"rendered":"
This shell script finds duplicate files in a given directory comparing their (md5) checksum. This means the content is checked and is strictly identical, rather than the filename or date of creation. This is usually useful to delete large files. ‘Find’ command option -size can help speeding up and finding the largest duplicate files. […]<\/p>\n","protected":false},"author":2,"featured_media":0,"comment_status":"open","ping_status":"closed","sticky":false,"template":"","format":"standard","meta":{"_monsterinsights_skip_tracking":false,"_monsterinsights_sitenote_active":false,"_monsterinsights_sitenote_note":"","_monsterinsights_sitenote_category":0},"categories":[11],"tags":[386,123],"yoast_head":"\n
Netexpertise - Find Duplicate Files with a Shell Script<\/title>\n<meta name=\"robots\" content=\"index, follow, max-snippet:-1, max-image-preview:large, max-video-preview:-1\" \/>\n<link rel=\"canonical\" href=\"http:\/\/www.netexpertise.eu\/en\/systems\/linux\/find-duplicate-files-shell-script.html\" \/>\n<meta property=\"og:locale\" content=\"en_US\" \/>\n<meta property=\"og:type\" content=\"article\" \/>\n<meta property=\"og:title\" content=\"Netexpertise - Find Duplicate Files with a Shell Script\" \/>\n<meta property=\"og:description\" content=\"This shell script finds duplicate files in a given directory comparing their (md5) checksum. This means the content is checked and is strictly identical, rather than the filename or date of creation. This is usually useful to delete large files. ‘Find’ command option -size can help speeding up and finding the largest duplicate files. […]\" \/>\n<meta property=\"og:url\" content=\"http:\/\/www.netexpertise.eu\/en\/systems\/linux\/find-duplicate-files-shell-script.html\" \/>\n<meta property=\"og:site_name\" content=\"Netexpertise\" \/>\n<meta property=\"article:published_time\" content=\"2009-01-21T21:02:33+00:00\" \/>\n<meta property=\"article:modified_time\" content=\"2021-10-06T20:42:40+00:00\" \/>\n<meta name=\"author\" content=\"dave\" \/>\n<meta name=\"twitter:card\" content=\"summary_large_image\" \/>\n<meta name=\"twitter:creator\" content=\"@netexpertise\" \/>\n<meta name=\"twitter:site\" content=\"@netexpertise\" \/>\n<script type=\"application\/ld+json\" class=\"yoast-schema-graph\">{\"@context\":\"https:\/\/schema.org\",\"@graph\":[{\"@type\":\"WebPage\",\"@id\":\"http:\/\/www.netexpertise.eu\/en\/systems\/linux\/find-duplicate-files-shell-script.html\",\"url\":\"http:\/\/www.netexpertise.eu\/en\/systems\/linux\/find-duplicate-files-shell-script.html\",\"name\":\"Netexpertise - Find Duplicate Files with a Shell Script\",\"isPartOf\":{\"@id\":\"http:\/\/www.netexpertise.eu\/en\/#website\"},\"datePublished\":\"2009-01-21T21:02:33+00:00\",\"dateModified\":\"2021-10-06T20:42:40+00:00\",\"author\":{\"@id\":\"http:\/\/www.netexpertise.eu\/en\/#\/schema\/person\/cb4cd666549d22e9070ec1cfc1a496fa\"},\"breadcrumb\":{\"@id\":\"http:\/\/www.netexpertise.eu\/en\/systems\/linux\/find-duplicate-files-shell-script.html#breadcrumb\"},\"inLanguage\":\"en-US\",\"potentialAction\":[{\"@type\":\"ReadAction\",\"target\":[\"http:\/\/www.netexpertise.eu\/en\/systems\/linux\/find-duplicate-files-shell-script.html\"]}]},{\"@type\":\"BreadcrumbList\",\"@id\":\"http:\/\/www.netexpertise.eu\/en\/systems\/linux\/find-duplicate-files-shell-script.html#breadcrumb\",\"itemListElement\":[{\"@type\":\"ListItem\",\"position\":1,\"name\":\"Home\",\"item\":\"http:\/\/www.netexpertise.eu\/en\"},{\"@type\":\"ListItem\",\"position\":2,\"name\":\"Find Duplicate Files with a Shell Script\"}]},{\"@type\":\"WebSite\",\"@id\":\"http:\/\/www.netexpertise.eu\/en\/#website\",\"url\":\"http:\/\/www.netexpertise.eu\/en\/\",\"name\":\"Netexpertise\",\"description\":\"Systems \/ Networks \/ DevOps\",\"potentialAction\":[{\"@type\":\"SearchAction\",\"target\":{\"@type\":\"EntryPoint\",\"urlTemplate\":\"http:\/\/www.netexpertise.eu\/en\/?s={search_term_string}\"},\"query-input\":\"required name=search_term_string\"}],\"inLanguage\":\"en-US\"},{\"@type\":\"Person\",\"@id\":\"http:\/\/www.netexpertise.eu\/en\/#\/schema\/person\/cb4cd666549d22e9070ec1cfc1a496fa\",\"name\":\"dave\",\"image\":{\"@type\":\"ImageObject\",\"inLanguage\":\"en-US\",\"@id\":\"http:\/\/www.netexpertise.eu\/en\/#\/schema\/person\/image\/\",\"url\":\"http:\/\/1.gravatar.com\/avatar\/1129916e1f4955bd632f27f836f64e55?s=96&d=mm&r=g\",\"contentUrl\":\"http:\/\/1.gravatar.com\/avatar\/1129916e1f4955bd632f27f836f64e55?s=96&d=mm&r=g\",\"caption\":\"dave\"},\"sameAs\":[\"http:\/\/www.netexpertise.eu\"]}]}<\/script>\n<!-- \/ Yoast SEO plugin. -->","yoast_head_json":{"title":"Netexpertise - Find Duplicate Files with a Shell Script","robots":{"index":"index","follow":"follow","max-snippet":"max-snippet:-1","max-image-preview":"max-image-preview:large","max-video-preview":"max-video-preview:-1"},"canonical":"http:\/\/www.netexpertise.eu\/en\/systems\/linux\/find-duplicate-files-shell-script.html","og_locale":"en_US","og_type":"article","og_title":"Netexpertise - Find Duplicate Files with a Shell Script","og_description":"This shell script finds duplicate files in a given directory comparing their (md5) checksum. This means the content is checked and is strictly identical, rather than the filename or date of creation. This is usually useful to delete large files. ‘Find’ command option -size can help speeding up and finding the largest duplicate files. […]","og_url":"http:\/\/www.netexpertise.eu\/en\/systems\/linux\/find-duplicate-files-shell-script.html","og_site_name":"Netexpertise","article_published_time":"2009-01-21T21:02:33+00:00","article_modified_time":"2021-10-06T20:42:40+00:00","author":"dave","twitter_card":"summary_large_image","twitter_creator":"@netexpertise","twitter_site":"@netexpertise","schema":{"@context":"https:\/\/schema.org","@graph":[{"@type":"WebPage","@id":"http:\/\/www.netexpertise.eu\/en\/systems\/linux\/find-duplicate-files-shell-script.html","url":"http:\/\/www.netexpertise.eu\/en\/systems\/linux\/find-duplicate-files-shell-script.html","name":"Netexpertise - Find Duplicate Files with a Shell Script","isPartOf":{"@id":"http:\/\/www.netexpertise.eu\/en\/#website"},"datePublished":"2009-01-21T21:02:33+00:00","dateModified":"2021-10-06T20:42:40+00:00","author":{"@id":"http:\/\/www.netexpertise.eu\/en\/#\/schema\/person\/cb4cd666549d22e9070ec1cfc1a496fa"},"breadcrumb":{"@id":"http:\/\/www.netexpertise.eu\/en\/systems\/linux\/find-duplicate-files-shell-script.html#breadcrumb"},"inLanguage":"en-US","potentialAction":[{"@type":"ReadAction","target":["http:\/\/www.netexpertise.eu\/en\/systems\/linux\/find-duplicate-files-shell-script.html"]}]},{"@type":"BreadcrumbList","@id":"http:\/\/www.netexpertise.eu\/en\/systems\/linux\/find-duplicate-files-shell-script.html#breadcrumb","itemListElement":[{"@type":"ListItem","position":1,"name":"Home","item":"http:\/\/www.netexpertise.eu\/en"},{"@type":"ListItem","position":2,"name":"Find Duplicate Files with a Shell Script"}]},{"@type":"WebSite","@id":"http:\/\/www.netexpertise.eu\/en\/#website","url":"http:\/\/www.netexpertise.eu\/en\/","name":"Netexpertise","description":"Systems \/ Networks \/ DevOps","potentialAction":[{"@type":"SearchAction","target":{"@type":"EntryPoint","urlTemplate":"http:\/\/www.netexpertise.eu\/en\/?s={search_term_string}"},"query-input":"required name=search_term_string"}],"inLanguage":"en-US"},{"@type":"Person","@id":"http:\/\/www.netexpertise.eu\/en\/#\/schema\/person\/cb4cd666549d22e9070ec1cfc1a496fa","name":"dave","image":{"@type":"ImageObject","inLanguage":"en-US","@id":"http:\/\/www.netexpertise.eu\/en\/#\/schema\/person\/image\/","url":"http:\/\/1.gravatar.com\/avatar\/1129916e1f4955bd632f27f836f64e55?s=96&d=mm&r=g","contentUrl":"http:\/\/1.gravatar.com\/avatar\/1129916e1f4955bd632f27f836f64e55?s=96&d=mm&r=g","caption":"dave"},"sameAs":["http:\/\/www.netexpertise.eu"]}]}},"_links":{"self":[{"href":"http:\/\/www.netexpertise.eu\/en\/wp-json\/wp\/v2\/posts\/260"}],"collection":[{"href":"http:\/\/www.netexpertise.eu\/en\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"http:\/\/www.netexpertise.eu\/en\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"http:\/\/www.netexpertise.eu\/en\/wp-json\/wp\/v2\/users\/2"}],"replies":[{"embeddable":true,"href":"http:\/\/www.netexpertise.eu\/en\/wp-json\/wp\/v2\/comments?post=260"}],"version-history":[{"count":0,"href":"http:\/\/www.netexpertise.eu\/en\/wp-json\/wp\/v2\/posts\/260\/revisions"}],"wp:attachment":[{"href":"http:\/\/www.netexpertise.eu\/en\/wp-json\/wp\/v2\/media?parent=260"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"http:\/\/www.netexpertise.eu\/en\/wp-json\/wp\/v2\/categories?post=260"},{"taxonomy":"post_tag","embeddable":true,"href":"http:\/\/www.netexpertise.eu\/en\/wp-json\/wp\/v2\/tags?post=260"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}