分享web开发知识

注册/登录|最近发布|今日推荐

主页 IT知识网页技术软件开发前端开发代码编程运营维护技术分享教程案例
当前位置:首页 > 运营维护

php表格--大数据处理

发布时间:2023-09-06 02:17责任编辑:熊小新关键词:暂无标签

参考来源1:https://blog.csdn.net/tim_phper/article/details/77581071

参考来源2:https://blog.csdn.net/qq_37682202/article/details/79426810

在数据不太多的时候,用xlsx表格导出导入还是可以的。

数据量很大时(5万条以上),用 PHPExcel 导出 xls 将十分缓慢且占用很大内存,最终造成运行超时或内存不足。

excel也是有脾气的呀!

  • 表数据限制:
Excel 2003及以下的版本。一张表最大支持65536行数据,256列。Excel 2007-2010版本。一张表最大支持1048576行,16384列。

也就是说你想几百万条轻轻松松一次性导入一张EXCEL表是不行的,你起码需要进行数据分割,保证数据不能超过104W一张表。

到数据的导入与导出,因为数据比较多,思来想去使用了CSV格式的文件,结合了网上各位大佬的文章,具体代码如下:

在thinkphp/library/think下新建Csv.php文件:

<?phpnamespace Think;class Csv{ ???//导出csv文件 ???public function put_csv($list,$title) ???{ ???????$file_name = "exam".time().".csv"; ???????header(‘Content-Type: application/vnd.ms-excel‘); ???????header(‘Content-Disposition: attachment;filename=‘.$file_name ); ???????header(‘Cache-Control: max-age=0‘); ???????$file = fopen(‘php://output‘,"a"); ???????$limit = 1000; ???????$calc = 0; ???????foreach ($title as $v){ ???????????$tit[] = iconv(‘UTF-8‘, ‘GB2312//IGNORE‘,$v); ???????} ???????fputcsv($file,$tit); ???????foreach ($list as $v){ ???????????$calc++; ???????????if($limit == $calc){ ???????????????ob_flush(); ???????????????flush(); ???????????????$calc = 0; ???????????} ???????????foreach($v as $t){ ???????????????$tarr[] = iconv(‘UTF-8‘, ‘GB2312//IGNORE‘,$t); ???????????} ???????????fputcsv($file,$tarr); ???????????unset($tarr); ???????} ???????unset($list); ???????fclose($file); ???????exit(); ???} ??????// csv导入,此格式每次最多可以处理1000条数据 ?????public function input_csv($csv_file) { ?????????$result_arr = array (); ?????????$i = 0; ?????????while($data_line = fgetcsv($csv_file,10000)) { ?????????????if ($i == 0) { ?????????????????$GLOBALS [‘csv_key_name_arr‘] = $data_line; ?????????????????$i ++; ?????????????????continue; ?????????????} ?????????????????foreach($GLOBALS[‘csv_key_name_arr‘] as $csv_key_num => $csv_key_name ) { ?????????????????$result_arr[$i][$csv_key_name] = $data_line[$csv_key_num]; ?????????????} ?????????????$i++; ?????????} ?????????return $result_arr; ?????} ?}?>

在控制器中调用:

<?phpnamespace app\index\controller;use think\Controller;use think\Request;use think\Db;use think\Paginator;use think\Validate;use think\Csv; ?//此处是csv文件实现的关键class Index extends Controller{ ???public function _initialize() ???{ ?????header("Content-type:text/html;charset=utf-8"); ???} ???????/* ????* CSV试题导出 ????*/ ???public function downQuestions() ???{ ??????????????$condition[‘qu_status‘] = array(‘gt‘,‘-1‘); ???????$csv = new Csv(); ?//实例化后才可以调用之前类文件定义好的方法 ???????$list = Db::table(‘gw_questions‘)->field(‘qu_name,qu_options,qu_answer,qu_describe,gw_topic.to_name,qu_time,te_name‘)->join(‘gw_topic‘,‘gw_topic.to_id=gw_questions.to_id‘)->join(‘gw_teacher‘,‘gw_teacher.te_id=gw_questions.te_id‘,‘LEFT‘)->where($condition)->select(); ???????$csv_title = array(‘题干内容‘,‘选项详情‘,‘参考答案‘,‘答案解析‘,‘题型‘,‘录入时间‘,‘录入人‘); ???????$csv->put_csv($list,$csv_title); ???} ???????/* ????* CSV试题导入 ????*/ ??????public function upQuestionsWrite() ???{ ???????// 获取表单上传文件 ???????$file = request()->file(‘examfile‘); ???????if(empty($file)) { ???????????$this->error(‘请选择上传文件‘); ???????} ???????// 移动到框架应用根目录/public/uploads/ 目录下 ???????$info = $file->move(ROOT_PATH.‘public‘.DS.‘upload‘); ????????//获取文件(日期/文件),$info->getFilename(); ???????$filename = ROOT_PATH.‘public‘.DS.‘upload/‘.$info->getSaveName(); ???????$handle = fopen($filename,‘r‘); ???????$csv = new Csv(); ?????????$result = $csv->input_csv($handle); // 解析csv ?????????$len_result = count($result); ?????????if($len_result == 0){ ?????????????$this->error(‘此文件中没有数据!‘); ???????} ?????????$data_values = ‘‘; ?????????for($i = 1;$i < $len_result+1;$i ++) { // 循环获取各字段值 ?????????????$arr = array_values($result[$i]); ?????????????$qu_name = iconv(‘gb2312‘,‘utf-8‘,$arr[0] ); // 中文转码 ?????????????$qu_options = iconv(‘gb2312‘,‘utf-8‘,$arr[1]); ?????????????$qu_answer = $arr[2]; ?????????????$qu_describe = iconv(‘gb2312‘,‘utf-8‘,$arr[3]); ???????????$to_id = $arr[4]; ????????????$te_id = 1; //添加试题的教师 ??????????????????????$data_values .= "(‘$qu_name‘,‘$qu_options‘,‘$qu_answer‘,‘$qu_describe‘,‘$to_id‘,‘$te_id‘),"; ?????????} ?????????$data_values = substr($data_values,0,- 1 ); // 去掉最后一个逗号 ?????????fclose($handle); // 关闭指针 ???????// 批量插入数据表中 ???????$result = DB::execute("insert into gw_questions (qu_name,qu_options,qu_answer,qu_describe,to_id,te_id) values $data_values" ); ???????if($result){ ???????????????????????$this->success(‘文件上传成功,数据已经导入!‘,‘exampaper‘,3); ???????}else{ ???????????// 上传失败获取错误信息 ???????????$this->error($file->getError()); ???????} ???} ??}?> 

模板页表单(此处我的项目中使用了框架,样式可以按自己的需求修改):

<form action="{:url(‘index/index/upQuestionsWrite‘)}" method="post" class="form form-horizontal" enctype="multipart/form-data" id="addform"> ???    <div class="row cl"> ???<label class="form-label col-xs-4 col-sm-2">选择试题文件:</label> ???<div class="btn-upload form-group"> ???    <input type="text" name="uploadfile" id="uploadfile" class="input-text upload-url radius" readonly> <a href="javascript:void();" class="btn btn-primary radius"><i class="Hui-iconfont"></i>浏览文件</a> ???????<input type="file" name="examfile" class="input-file" multiple> ???????????????????</div>     ???<a class="btn btn-success btn-submit">导入试题</a> ???</div> ???</form>
<script type="text/javascript">$(function(){ ????????//表单提交时判断是否有文件存在 ???$(".btn-submit").click(function(){ ???????if($("input[name=‘uploadfile‘]").val() == ‘‘){ ???????????alert("请选择CSV文件!"); ???????????return false; ???}else{ ???    $("#addform").submit(); ???} ???}); ????});</script>

php表格--大数据处理

原文地址:https://www.cnblogs.com/laijinquan/p/9747011.html

知识推荐

我的编程学习网——分享web前端后端开发技术知识。 垃圾信息处理邮箱 tousu563@163.com 网站地图
icp备案号 闽ICP备2023006418号-8 不良信息举报平台 互联网安全管理备案 Copyright 2023 www.wodecom.cn All Rights Reserved